Skip to content

Commit

Permalink
Require fork for multiprocessing (#100)
Browse files Browse the repository at this point in the history
This avoids errors with pickling the target module when running in a subprocess.

Fixes #86.
  • Loading branch information
arahlin authored Apr 7, 2023
1 parent e07db4e commit 8e4817d
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 11 deletions.
13 changes: 6 additions & 7 deletions core/python/multiprocess.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
import sys
if sys.version_info[:2] > (3, 7):
raise ImportError("Multiprocessing is disabled for python versions > 3.7")

from multiprocessing import Process
from multiprocessing import get_context
import socket, pickle, errno, struct, time

from spt3g.core import G3FrameType, G3Frame

class Subproc(Process):
# Require fork to avoid pickling errors
ctx = get_context("fork")

class Subproc(ctx.Process):
'''
Run a module in a subprocess, using python multiprocessing to proxy
frames to it. If more than maxqueuelen frames are queued on the
Expand All @@ -18,7 +17,7 @@ def __init__(self, target=None, name=None, maxqueuelen=50):
Set up a multiprocessing shim for the pipeline module <target>
under the process name <name> (can be None).
'''
Process.__init__(self, name=name)
ctx.Process.__init__(self, name=name)
self.targetmod = target

self.queue = socket.socketpair()
Expand Down
4 changes: 0 additions & 4 deletions core/tests/multiproc.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,6 @@ def checkinfo(fr):
m += 1

if __name__ == '__main__':
if sys.version_info[:2] > (3, 7):
print('Subprocess option is disabled for python versions > 3.7')
raise SystemExit

pipe = core.G3Pipeline()
pipe.Add(core.G3InfiniteSource, type=core.G3FrameType.Timepoint, n=10)
pipe.Add(addinfo, subprocess=True)
Expand Down

0 comments on commit 8e4817d

Please sign in to comment.