diff --git a/demo.py b/demo.py deleted file mode 100644 index fd2af9d..0000000 --- a/demo.py +++ /dev/null @@ -1,14 +0,0 @@ -from fork_map import fork_map - -result = fork_map(lambda x: x * 2, range(5)) -print(result) - -## ----------------------------------- - -from multiprocessing.pool import Pool - -# This fails, as lambdas aren't pickleable. -with Pool() as p: - result = p.map(lambda x: x * 2, range(5)) - -print(result) diff --git a/fork_map.wpr b/fork_map.wpr index 40c14f7..204b4ec 100644 --- a/fork_map.wpr +++ b/fork_map.wpr @@ -4,6 +4,16 @@ # Wing project file # ################################################################## [project attributes] +console.toolbox = [{'autosave': False, + 'id': 'cmd-JWEPUsuAbCaNufpa', + 'io_encoding': None, + 'key_binding': None, + 'line_mode': True, + 'loc': '/home/rutherford/code/fork_map/fork_map/demos/fork.py', + 'pseudo_tty': False, + 'raise_panel': True, + 'shared': False, + 'title': None}] proj.directory-list = [{'dirloc': loc('fork_map'), 'excludes': (), 'filter': '*', diff --git a/fork_map/demos/deadlock.py b/fork_map/demos/deadlock.py new file mode 100644 index 0000000..4e45fac --- /dev/null +++ b/fork_map/demos/deadlock.py @@ -0,0 +1,23 @@ +"""The below code causes a deadlock on python <3.14""" + +import os +import time +from concurrent.futures import ProcessPoolExecutor +import threading + +lock = threading.Lock() + + +def process_items(name): + print(f"{name}: acquiring lock") + with lock: + print(f"{name}: has lock") + time.sleep(0.1) + print(f"{name}: released lock") + + +t = threading.Thread(target=process_items, args=("Thread",)) +t.start() + +with ProcessPoolExecutor() as e: + e.submit(process_items, "Process") diff --git a/fork_map/demos/demo.py b/fork_map/demos/demo.py new file mode 100644 index 0000000..fcf011d --- /dev/null +++ b/fork_map/demos/demo.py @@ -0,0 +1,25 @@ +import os + +# Todo fix import paths +from fork_map.fork_map import fork_map + +result = fork_map(lambda x: x * 2, range(5)) +print(result) + +## ----------------------------------- + +from multiprocessing.pool import Pool + +# This fails, as lambdas aren't pickleable. +# with Pool() as p: +# result = p.map(lambda x: x * 2, range(5)) + +# print(result) + + +# ---- + +LETTERS = "abcde" + +for r in fork_map(lambda idx: f"{os.getpid()} got letter {LETTERS[idx]}", [0, 1, 3]): + print(r) diff --git a/fork_map/demos/fork.py b/fork_map/demos/fork.py new file mode 100644 index 0000000..ffda263 --- /dev/null +++ b/fork_map/demos/fork.py @@ -0,0 +1,7 @@ +import os + +pid = os.fork() +if pid: + print('I am the parent') +else: + print('I am the child') diff --git a/fork_map/demos/fork_vs_spawn.py b/fork_map/demos/fork_vs_spawn.py new file mode 100644 index 0000000..4ca324e --- /dev/null +++ b/fork_map/demos/fork_vs_spawn.py @@ -0,0 +1,19 @@ +"""The below fails unless start method is fork""" + +import os +import multiprocessing +from concurrent.futures import ProcessPoolExecutor + + +def get_letter(idx: int) -> str: + pid = os.getpid() + return f"{pid=} got letter {LETTERS[idx]}" + + +if __name__ == "__main__": + multiprocessing.set_start_method("spawn") + LETTERS = "abcde" + + with ProcessPoolExecutor() as e: + for r in e.map(get_letter, [0, 1, 3]): + print(r) diff --git a/fork_map/fork_map.py b/fork_map/fork_map.py index 92e0ffc..8ab852b 100644 --- a/fork_map/fork_map.py +++ b/fork_map/fork_map.py @@ -43,12 +43,13 @@ def _process_in_fork( try: r = func(*args, **kwargs) - # pickle here, so that we can't crash with pickle errors in the finally clause + # pickle here, so that we can't crash with pickle errors in the finally clause. pickled_r = pickle.dumps(r) result = make_result(result=pickled_r) except Exception as e: try: - # In case func does something stupid like raising an unpicklable exception + # In case func does something untoward like raising an unpicklable + # exception. pickled_exception = pickle.dumps(e) except AttributeError: pickled_exception = pickle.dumps(