summaryrefslogtreecommitdiffstats
path: root/contrib/tools/python3/src/Lib/multiprocessing
diff options
context:
space:
mode:
authororivej <[email protected]>2022-02-10 16:44:49 +0300
committerDaniil Cherednik <[email protected]>2022-02-10 16:44:49 +0300
commit718c552901d703c502ccbefdfc3c9028d608b947 (patch)
tree46534a98bbefcd7b1f3faa5b52c138ab27db75b7 /contrib/tools/python3/src/Lib/multiprocessing
parente9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (diff)
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/python3/src/Lib/multiprocessing')
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/__init__.py70
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/connection.py1900
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/context.py700
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/dummy/__init__.py250
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/dummy/connection.py150
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/forkserver.py636
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/heap.py492
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/managers.py2378
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/pool.py1514
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/popen_fork.py150
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/popen_forkserver.py138
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_posix.py126
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_win32.py246
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/process.py720
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/queues.py716
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/reduction.py548
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/resource_sharer.py302
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/sharedctypes.py480
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/spawn.py560
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/synchronize.py778
-rw-r--r--contrib/tools/python3/src/Lib/multiprocessing/util.py862
21 files changed, 6858 insertions, 6858 deletions
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/__init__.py b/contrib/tools/python3/src/Lib/multiprocessing/__init__.py
index 8336f381dec..a8e9c54bd0b 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/__init__.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/__init__.py
@@ -1,37 +1,37 @@
-#
-# Package analogous to 'threading.py' but using processes
-#
-# multiprocessing/__init__.py
-#
-# This package is intended to duplicate the functionality (and much of
-# the API) of threading.py but uses processes instead of threads. A
-# subpackage 'multiprocessing.dummy' has the same API but is a simple
-# wrapper for 'threading'.
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-import sys
-from . import context
-
-#
-# Copy stuff from default context
-#
-
+#
+# Package analogous to 'threading.py' but using processes
+#
+# multiprocessing/__init__.py
+#
+# This package is intended to duplicate the functionality (and much of
+# the API) of threading.py but uses processes instead of threads. A
+# subpackage 'multiprocessing.dummy' has the same API but is a simple
+# wrapper for 'threading'.
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import sys
+from . import context
+
+#
+# Copy stuff from default context
+#
+
__all__ = [x for x in dir(context._default_context) if not x.startswith('_')]
globals().update((name, getattr(context._default_context, name)) for name in __all__)
-
-#
-# XXX These should not really be documented or public.
-#
-
-SUBDEBUG = 5
-SUBWARNING = 25
-
-#
-# Alias for main module -- will be reset by bootstrapping child processes
-#
-
-if '__main__' in sys.modules:
- sys.modules['__mp_main__'] = sys.modules['__main__']
+
+#
+# XXX These should not really be documented or public.
+#
+
+SUBDEBUG = 5
+SUBWARNING = 25
+
+#
+# Alias for main module -- will be reset by bootstrapping child processes
+#
+
+if '__main__' in sys.modules:
+ sys.modules['__mp_main__'] = sys.modules['__main__']
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/connection.py b/contrib/tools/python3/src/Lib/multiprocessing/connection.py
index 510e4b5aba4..e234f368cfe 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/connection.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/connection.py
@@ -1,406 +1,406 @@
-#
-# A higher level module for using sockets (or Windows named pipes)
-#
-# multiprocessing/connection.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
-
-import io
-import os
-import sys
-import socket
-import struct
-import time
-import tempfile
-import itertools
-
-import _multiprocessing
-
-from . import util
-
-from . import AuthenticationError, BufferTooShort
-from .context import reduction
-_ForkingPickler = reduction.ForkingPickler
-
-try:
- import _winapi
- from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
-except ImportError:
- if sys.platform == 'win32':
- raise
- _winapi = None
-
-#
-#
-#
-
-BUFSIZE = 8192
-# A very generous timeout when it comes to local connections...
-CONNECTION_TIMEOUT = 20.
-
-_mmap_counter = itertools.count()
-
-default_family = 'AF_INET'
-families = ['AF_INET']
-
-if hasattr(socket, 'AF_UNIX'):
- default_family = 'AF_UNIX'
- families += ['AF_UNIX']
-
-if sys.platform == 'win32':
- default_family = 'AF_PIPE'
- families += ['AF_PIPE']
-
-
-def _init_timeout(timeout=CONNECTION_TIMEOUT):
- return time.monotonic() + timeout
-
-def _check_timeout(t):
- return time.monotonic() > t
-
-#
-#
-#
-
-def arbitrary_address(family):
- '''
- Return an arbitrary free address for the given family
- '''
- if family == 'AF_INET':
- return ('localhost', 0)
- elif family == 'AF_UNIX':
+#
+# A higher level module for using sockets (or Windows named pipes)
+#
+# multiprocessing/connection.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
+
+import io
+import os
+import sys
+import socket
+import struct
+import time
+import tempfile
+import itertools
+
+import _multiprocessing
+
+from . import util
+
+from . import AuthenticationError, BufferTooShort
+from .context import reduction
+_ForkingPickler = reduction.ForkingPickler
+
+try:
+ import _winapi
+ from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
+except ImportError:
+ if sys.platform == 'win32':
+ raise
+ _winapi = None
+
+#
+#
+#
+
+BUFSIZE = 8192
+# A very generous timeout when it comes to local connections...
+CONNECTION_TIMEOUT = 20.
+
+_mmap_counter = itertools.count()
+
+default_family = 'AF_INET'
+families = ['AF_INET']
+
+if hasattr(socket, 'AF_UNIX'):
+ default_family = 'AF_UNIX'
+ families += ['AF_UNIX']
+
+if sys.platform == 'win32':
+ default_family = 'AF_PIPE'
+ families += ['AF_PIPE']
+
+
+def _init_timeout(timeout=CONNECTION_TIMEOUT):
+ return time.monotonic() + timeout
+
+def _check_timeout(t):
+ return time.monotonic() > t
+
+#
+#
+#
+
+def arbitrary_address(family):
+ '''
+ Return an arbitrary free address for the given family
+ '''
+ if family == 'AF_INET':
+ return ('localhost', 0)
+ elif family == 'AF_UNIX':
# Prefer abstract sockets if possible to avoid problems with the address
# size. When coding portable applications, some implementations have
# sun_path as short as 92 bytes in the sockaddr_un struct.
if util.abstract_sockets_supported:
return f"\0listener-{os.getpid()}-{next(_mmap_counter)}"
- return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
- elif family == 'AF_PIPE':
- return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
- (os.getpid(), next(_mmap_counter)), dir="")
- else:
- raise ValueError('unrecognized family')
-
-def _validate_family(family):
- '''
- Checks if the family is valid for the current environment.
- '''
- if sys.platform != 'win32' and family == 'AF_PIPE':
- raise ValueError('Family %s is not recognized.' % family)
-
- if sys.platform == 'win32' and family == 'AF_UNIX':
- # double check
- if not hasattr(socket, family):
- raise ValueError('Family %s is not recognized.' % family)
-
-def address_type(address):
- '''
- Return the types of the address
-
- This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
- '''
- if type(address) == tuple:
- return 'AF_INET'
- elif type(address) is str and address.startswith('\\\\'):
- return 'AF_PIPE'
+ return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
+ elif family == 'AF_PIPE':
+ return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
+ (os.getpid(), next(_mmap_counter)), dir="")
+ else:
+ raise ValueError('unrecognized family')
+
+def _validate_family(family):
+ '''
+ Checks if the family is valid for the current environment.
+ '''
+ if sys.platform != 'win32' and family == 'AF_PIPE':
+ raise ValueError('Family %s is not recognized.' % family)
+
+ if sys.platform == 'win32' and family == 'AF_UNIX':
+ # double check
+ if not hasattr(socket, family):
+ raise ValueError('Family %s is not recognized.' % family)
+
+def address_type(address):
+ '''
+ Return the types of the address
+
+ This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
+ '''
+ if type(address) == tuple:
+ return 'AF_INET'
+ elif type(address) is str and address.startswith('\\\\'):
+ return 'AF_PIPE'
elif type(address) is str or util.is_abstract_socket_namespace(address):
- return 'AF_UNIX'
- else:
- raise ValueError('address type of %r unrecognized' % address)
-
-#
-# Connection classes
-#
-
-class _ConnectionBase:
- _handle = None
-
- def __init__(self, handle, readable=True, writable=True):
- handle = handle.__index__()
- if handle < 0:
- raise ValueError("invalid handle")
- if not readable and not writable:
- raise ValueError(
- "at least one of `readable` and `writable` must be True")
- self._handle = handle
- self._readable = readable
- self._writable = writable
-
- # XXX should we use util.Finalize instead of a __del__?
-
- def __del__(self):
- if self._handle is not None:
- self._close()
-
- def _check_closed(self):
- if self._handle is None:
- raise OSError("handle is closed")
-
- def _check_readable(self):
- if not self._readable:
- raise OSError("connection is write-only")
-
- def _check_writable(self):
- if not self._writable:
- raise OSError("connection is read-only")
-
- def _bad_message_length(self):
- if self._writable:
- self._readable = False
- else:
- self.close()
- raise OSError("bad message length")
-
- @property
- def closed(self):
- """True if the connection is closed"""
- return self._handle is None
-
- @property
- def readable(self):
- """True if the connection is readable"""
- return self._readable
-
- @property
- def writable(self):
- """True if the connection is writable"""
- return self._writable
-
- def fileno(self):
- """File descriptor or handle of the connection"""
- self._check_closed()
- return self._handle
-
- def close(self):
- """Close the connection"""
- if self._handle is not None:
- try:
- self._close()
- finally:
- self._handle = None
-
- def send_bytes(self, buf, offset=0, size=None):
- """Send the bytes data from a bytes-like object"""
- self._check_closed()
- self._check_writable()
- m = memoryview(buf)
- # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
- if m.itemsize > 1:
- m = memoryview(bytes(m))
- n = len(m)
- if offset < 0:
- raise ValueError("offset is negative")
- if n < offset:
- raise ValueError("buffer length < offset")
- if size is None:
- size = n - offset
- elif size < 0:
- raise ValueError("size is negative")
- elif offset + size > n:
- raise ValueError("buffer length < offset + size")
- self._send_bytes(m[offset:offset + size])
-
- def send(self, obj):
- """Send a (picklable) object"""
- self._check_closed()
- self._check_writable()
- self._send_bytes(_ForkingPickler.dumps(obj))
-
- def recv_bytes(self, maxlength=None):
- """
- Receive bytes data as a bytes object.
- """
- self._check_closed()
- self._check_readable()
- if maxlength is not None and maxlength < 0:
- raise ValueError("negative maxlength")
- buf = self._recv_bytes(maxlength)
- if buf is None:
- self._bad_message_length()
- return buf.getvalue()
-
- def recv_bytes_into(self, buf, offset=0):
- """
- Receive bytes data into a writeable bytes-like object.
- Return the number of bytes read.
- """
- self._check_closed()
- self._check_readable()
- with memoryview(buf) as m:
- # Get bytesize of arbitrary buffer
- itemsize = m.itemsize
- bytesize = itemsize * len(m)
- if offset < 0:
- raise ValueError("negative offset")
- elif offset > bytesize:
- raise ValueError("offset too large")
- result = self._recv_bytes()
- size = result.tell()
- if bytesize < offset + size:
- raise BufferTooShort(result.getvalue())
- # Message can fit in dest
- result.seek(0)
- result.readinto(m[offset // itemsize :
- (offset + size) // itemsize])
- return size
-
- def recv(self):
- """Receive a (picklable) object"""
- self._check_closed()
- self._check_readable()
- buf = self._recv_bytes()
- return _ForkingPickler.loads(buf.getbuffer())
-
- def poll(self, timeout=0.0):
- """Whether there is any input available to be read"""
- self._check_closed()
- self._check_readable()
- return self._poll(timeout)
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- self.close()
-
-
-if _winapi:
-
- class PipeConnection(_ConnectionBase):
- """
- Connection class based on a Windows named pipe.
- Overlapped I/O is used, so the handles must have been created
- with FILE_FLAG_OVERLAPPED.
- """
- _got_empty_message = False
-
- def _close(self, _CloseHandle=_winapi.CloseHandle):
- _CloseHandle(self._handle)
-
- def _send_bytes(self, buf):
- ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
- try:
- if err == _winapi.ERROR_IO_PENDING:
- waitres = _winapi.WaitForMultipleObjects(
- [ov.event], False, INFINITE)
- assert waitres == WAIT_OBJECT_0
- except:
- ov.cancel()
- raise
- finally:
- nwritten, err = ov.GetOverlappedResult(True)
- assert err == 0
- assert nwritten == len(buf)
-
- def _recv_bytes(self, maxsize=None):
- if self._got_empty_message:
- self._got_empty_message = False
- return io.BytesIO()
- else:
- bsize = 128 if maxsize is None else min(maxsize, 128)
- try:
- ov, err = _winapi.ReadFile(self._handle, bsize,
- overlapped=True)
- try:
- if err == _winapi.ERROR_IO_PENDING:
- waitres = _winapi.WaitForMultipleObjects(
- [ov.event], False, INFINITE)
- assert waitres == WAIT_OBJECT_0
- except:
- ov.cancel()
- raise
- finally:
- nread, err = ov.GetOverlappedResult(True)
- if err == 0:
- f = io.BytesIO()
- f.write(ov.getbuffer())
- return f
- elif err == _winapi.ERROR_MORE_DATA:
- return self._get_more_data(ov, maxsize)
- except OSError as e:
- if e.winerror == _winapi.ERROR_BROKEN_PIPE:
- raise EOFError
- else:
- raise
- raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
-
- def _poll(self, timeout):
- if (self._got_empty_message or
- _winapi.PeekNamedPipe(self._handle)[0] != 0):
- return True
- return bool(wait([self], timeout))
-
- def _get_more_data(self, ov, maxsize):
- buf = ov.getbuffer()
- f = io.BytesIO()
- f.write(buf)
- left = _winapi.PeekNamedPipe(self._handle)[1]
- assert left > 0
- if maxsize is not None and len(buf) + left > maxsize:
- self._bad_message_length()
- ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
- rbytes, err = ov.GetOverlappedResult(True)
- assert err == 0
- assert rbytes == left
- f.write(ov.getbuffer())
- return f
-
-
-class Connection(_ConnectionBase):
- """
- Connection class based on an arbitrary file descriptor (Unix only), or
- a socket handle (Windows).
- """
-
- if _winapi:
- def _close(self, _close=_multiprocessing.closesocket):
- _close(self._handle)
- _write = _multiprocessing.send
- _read = _multiprocessing.recv
- else:
- def _close(self, _close=os.close):
- _close(self._handle)
- _write = os.write
- _read = os.read
-
- def _send(self, buf, write=_write):
- remaining = len(buf)
- while True:
- n = write(self._handle, buf)
- remaining -= n
- if remaining == 0:
- break
- buf = buf[n:]
-
- def _recv(self, size, read=_read):
- buf = io.BytesIO()
- handle = self._handle
- remaining = size
- while remaining > 0:
- chunk = read(handle, remaining)
- n = len(chunk)
- if n == 0:
- if remaining == size:
- raise EOFError
- else:
- raise OSError("got end of file during message")
- buf.write(chunk)
- remaining -= n
- return buf
-
- def _send_bytes(self, buf):
- n = len(buf)
+ return 'AF_UNIX'
+ else:
+ raise ValueError('address type of %r unrecognized' % address)
+
+#
+# Connection classes
+#
+
+class _ConnectionBase:
+ _handle = None
+
+ def __init__(self, handle, readable=True, writable=True):
+ handle = handle.__index__()
+ if handle < 0:
+ raise ValueError("invalid handle")
+ if not readable and not writable:
+ raise ValueError(
+ "at least one of `readable` and `writable` must be True")
+ self._handle = handle
+ self._readable = readable
+ self._writable = writable
+
+ # XXX should we use util.Finalize instead of a __del__?
+
+ def __del__(self):
+ if self._handle is not None:
+ self._close()
+
+ def _check_closed(self):
+ if self._handle is None:
+ raise OSError("handle is closed")
+
+ def _check_readable(self):
+ if not self._readable:
+ raise OSError("connection is write-only")
+
+ def _check_writable(self):
+ if not self._writable:
+ raise OSError("connection is read-only")
+
+ def _bad_message_length(self):
+ if self._writable:
+ self._readable = False
+ else:
+ self.close()
+ raise OSError("bad message length")
+
+ @property
+ def closed(self):
+ """True if the connection is closed"""
+ return self._handle is None
+
+ @property
+ def readable(self):
+ """True if the connection is readable"""
+ return self._readable
+
+ @property
+ def writable(self):
+ """True if the connection is writable"""
+ return self._writable
+
+ def fileno(self):
+ """File descriptor or handle of the connection"""
+ self._check_closed()
+ return self._handle
+
+ def close(self):
+ """Close the connection"""
+ if self._handle is not None:
+ try:
+ self._close()
+ finally:
+ self._handle = None
+
+ def send_bytes(self, buf, offset=0, size=None):
+ """Send the bytes data from a bytes-like object"""
+ self._check_closed()
+ self._check_writable()
+ m = memoryview(buf)
+ # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
+ if m.itemsize > 1:
+ m = memoryview(bytes(m))
+ n = len(m)
+ if offset < 0:
+ raise ValueError("offset is negative")
+ if n < offset:
+ raise ValueError("buffer length < offset")
+ if size is None:
+ size = n - offset
+ elif size < 0:
+ raise ValueError("size is negative")
+ elif offset + size > n:
+ raise ValueError("buffer length < offset + size")
+ self._send_bytes(m[offset:offset + size])
+
+ def send(self, obj):
+ """Send a (picklable) object"""
+ self._check_closed()
+ self._check_writable()
+ self._send_bytes(_ForkingPickler.dumps(obj))
+
+ def recv_bytes(self, maxlength=None):
+ """
+ Receive bytes data as a bytes object.
+ """
+ self._check_closed()
+ self._check_readable()
+ if maxlength is not None and maxlength < 0:
+ raise ValueError("negative maxlength")
+ buf = self._recv_bytes(maxlength)
+ if buf is None:
+ self._bad_message_length()
+ return buf.getvalue()
+
+ def recv_bytes_into(self, buf, offset=0):
+ """
+ Receive bytes data into a writeable bytes-like object.
+ Return the number of bytes read.
+ """
+ self._check_closed()
+ self._check_readable()
+ with memoryview(buf) as m:
+ # Get bytesize of arbitrary buffer
+ itemsize = m.itemsize
+ bytesize = itemsize * len(m)
+ if offset < 0:
+ raise ValueError("negative offset")
+ elif offset > bytesize:
+ raise ValueError("offset too large")
+ result = self._recv_bytes()
+ size = result.tell()
+ if bytesize < offset + size:
+ raise BufferTooShort(result.getvalue())
+ # Message can fit in dest
+ result.seek(0)
+ result.readinto(m[offset // itemsize :
+ (offset + size) // itemsize])
+ return size
+
+ def recv(self):
+ """Receive a (picklable) object"""
+ self._check_closed()
+ self._check_readable()
+ buf = self._recv_bytes()
+ return _ForkingPickler.loads(buf.getbuffer())
+
+ def poll(self, timeout=0.0):
+ """Whether there is any input available to be read"""
+ self._check_closed()
+ self._check_readable()
+ return self._poll(timeout)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ self.close()
+
+
+if _winapi:
+
+ class PipeConnection(_ConnectionBase):
+ """
+ Connection class based on a Windows named pipe.
+ Overlapped I/O is used, so the handles must have been created
+ with FILE_FLAG_OVERLAPPED.
+ """
+ _got_empty_message = False
+
+ def _close(self, _CloseHandle=_winapi.CloseHandle):
+ _CloseHandle(self._handle)
+
+ def _send_bytes(self, buf):
+ ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
+ try:
+ if err == _winapi.ERROR_IO_PENDING:
+ waitres = _winapi.WaitForMultipleObjects(
+ [ov.event], False, INFINITE)
+ assert waitres == WAIT_OBJECT_0
+ except:
+ ov.cancel()
+ raise
+ finally:
+ nwritten, err = ov.GetOverlappedResult(True)
+ assert err == 0
+ assert nwritten == len(buf)
+
+ def _recv_bytes(self, maxsize=None):
+ if self._got_empty_message:
+ self._got_empty_message = False
+ return io.BytesIO()
+ else:
+ bsize = 128 if maxsize is None else min(maxsize, 128)
+ try:
+ ov, err = _winapi.ReadFile(self._handle, bsize,
+ overlapped=True)
+ try:
+ if err == _winapi.ERROR_IO_PENDING:
+ waitres = _winapi.WaitForMultipleObjects(
+ [ov.event], False, INFINITE)
+ assert waitres == WAIT_OBJECT_0
+ except:
+ ov.cancel()
+ raise
+ finally:
+ nread, err = ov.GetOverlappedResult(True)
+ if err == 0:
+ f = io.BytesIO()
+ f.write(ov.getbuffer())
+ return f
+ elif err == _winapi.ERROR_MORE_DATA:
+ return self._get_more_data(ov, maxsize)
+ except OSError as e:
+ if e.winerror == _winapi.ERROR_BROKEN_PIPE:
+ raise EOFError
+ else:
+ raise
+ raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
+
+ def _poll(self, timeout):
+ if (self._got_empty_message or
+ _winapi.PeekNamedPipe(self._handle)[0] != 0):
+ return True
+ return bool(wait([self], timeout))
+
+ def _get_more_data(self, ov, maxsize):
+ buf = ov.getbuffer()
+ f = io.BytesIO()
+ f.write(buf)
+ left = _winapi.PeekNamedPipe(self._handle)[1]
+ assert left > 0
+ if maxsize is not None and len(buf) + left > maxsize:
+ self._bad_message_length()
+ ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
+ rbytes, err = ov.GetOverlappedResult(True)
+ assert err == 0
+ assert rbytes == left
+ f.write(ov.getbuffer())
+ return f
+
+
+class Connection(_ConnectionBase):
+ """
+ Connection class based on an arbitrary file descriptor (Unix only), or
+ a socket handle (Windows).
+ """
+
+ if _winapi:
+ def _close(self, _close=_multiprocessing.closesocket):
+ _close(self._handle)
+ _write = _multiprocessing.send
+ _read = _multiprocessing.recv
+ else:
+ def _close(self, _close=os.close):
+ _close(self._handle)
+ _write = os.write
+ _read = os.read
+
+ def _send(self, buf, write=_write):
+ remaining = len(buf)
+ while True:
+ n = write(self._handle, buf)
+ remaining -= n
+ if remaining == 0:
+ break
+ buf = buf[n:]
+
+ def _recv(self, size, read=_read):
+ buf = io.BytesIO()
+ handle = self._handle
+ remaining = size
+ while remaining > 0:
+ chunk = read(handle, remaining)
+ n = len(chunk)
+ if n == 0:
+ if remaining == size:
+ raise EOFError
+ else:
+ raise OSError("got end of file during message")
+ buf.write(chunk)
+ remaining -= n
+ return buf
+
+ def _send_bytes(self, buf):
+ n = len(buf)
if n > 0x7fffffff:
pre_header = struct.pack("!i", -1)
header = struct.pack("!Q", n)
self._send(pre_header)
- self._send(header)
- self._send(buf)
- else:
+ self._send(header)
+ self._send(buf)
+ else:
# For wire compatibility with 3.7 and lower
header = struct.pack("!i", n)
if n > 16384:
@@ -414,565 +414,565 @@ class Connection(_ConnectionBase):
# Also note we want to avoid sending a 0-length buffer separately,
# to avoid "broken pipe" errors if the other end closed the pipe.
self._send(header + buf)
-
- def _recv_bytes(self, maxsize=None):
- buf = self._recv(4)
- size, = struct.unpack("!i", buf.getvalue())
+
+ def _recv_bytes(self, maxsize=None):
+ buf = self._recv(4)
+ size, = struct.unpack("!i", buf.getvalue())
if size == -1:
buf = self._recv(8)
size, = struct.unpack("!Q", buf.getvalue())
- if maxsize is not None and size > maxsize:
- return None
- return self._recv(size)
-
- def _poll(self, timeout):
- r = wait([self], timeout)
- return bool(r)
-
-
-#
-# Public functions
-#
-
-class Listener(object):
- '''
- Returns a listener object.
-
- This is a wrapper for a bound socket which is 'listening' for
- connections, or for a Windows named pipe.
- '''
- def __init__(self, address=None, family=None, backlog=1, authkey=None):
- family = family or (address and address_type(address)) \
- or default_family
- address = address or arbitrary_address(family)
-
- _validate_family(family)
- if family == 'AF_PIPE':
- self._listener = PipeListener(address, backlog)
- else:
- self._listener = SocketListener(address, family, backlog)
-
- if authkey is not None and not isinstance(authkey, bytes):
- raise TypeError('authkey should be a byte string')
-
- self._authkey = authkey
-
- def accept(self):
- '''
- Accept a connection on the bound socket or named pipe of `self`.
-
- Returns a `Connection` object.
- '''
- if self._listener is None:
- raise OSError('listener is closed')
- c = self._listener.accept()
- if self._authkey:
- deliver_challenge(c, self._authkey)
- answer_challenge(c, self._authkey)
- return c
-
- def close(self):
- '''
- Close the bound socket or named pipe of `self`.
- '''
- listener = self._listener
- if listener is not None:
- self._listener = None
- listener.close()
-
- @property
- def address(self):
- return self._listener._address
-
- @property
- def last_accepted(self):
- return self._listener._last_accepted
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- self.close()
-
-
-def Client(address, family=None, authkey=None):
- '''
- Returns a connection to the address of a `Listener`
- '''
- family = family or address_type(address)
- _validate_family(family)
- if family == 'AF_PIPE':
- c = PipeClient(address)
- else:
- c = SocketClient(address)
-
- if authkey is not None and not isinstance(authkey, bytes):
- raise TypeError('authkey should be a byte string')
-
- if authkey is not None:
- answer_challenge(c, authkey)
- deliver_challenge(c, authkey)
-
- return c
-
-
-if sys.platform != 'win32':
-
- def Pipe(duplex=True):
- '''
- Returns pair of connection objects at either end of a pipe
- '''
- if duplex:
- s1, s2 = socket.socketpair()
- s1.setblocking(True)
- s2.setblocking(True)
- c1 = Connection(s1.detach())
- c2 = Connection(s2.detach())
- else:
- fd1, fd2 = os.pipe()
- c1 = Connection(fd1, writable=False)
- c2 = Connection(fd2, readable=False)
-
- return c1, c2
-
-else:
-
- def Pipe(duplex=True):
- '''
- Returns pair of connection objects at either end of a pipe
- '''
- address = arbitrary_address('AF_PIPE')
- if duplex:
- openmode = _winapi.PIPE_ACCESS_DUPLEX
- access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
- obsize, ibsize = BUFSIZE, BUFSIZE
- else:
- openmode = _winapi.PIPE_ACCESS_INBOUND
- access = _winapi.GENERIC_WRITE
- obsize, ibsize = 0, BUFSIZE
-
- h1 = _winapi.CreateNamedPipe(
- address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
- _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
- _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
- _winapi.PIPE_WAIT,
- 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
- # default security descriptor: the handle cannot be inherited
- _winapi.NULL
- )
- h2 = _winapi.CreateFile(
- address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
- _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
- )
- _winapi.SetNamedPipeHandleState(
- h2, _winapi.PIPE_READMODE_MESSAGE, None, None
- )
-
- overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
- _, err = overlapped.GetOverlappedResult(True)
- assert err == 0
-
- c1 = PipeConnection(h1, writable=duplex)
- c2 = PipeConnection(h2, readable=duplex)
-
- return c1, c2
-
-#
-# Definitions for connections based on sockets
-#
-
-class SocketListener(object):
- '''
- Representation of a socket which is bound to an address and listening
- '''
- def __init__(self, address, family, backlog=1):
- self._socket = socket.socket(getattr(socket, family))
- try:
- # SO_REUSEADDR has different semantics on Windows (issue #2550).
- if os.name == 'posix':
- self._socket.setsockopt(socket.SOL_SOCKET,
- socket.SO_REUSEADDR, 1)
- self._socket.setblocking(True)
- self._socket.bind(address)
- self._socket.listen(backlog)
- self._address = self._socket.getsockname()
- except OSError:
- self._socket.close()
- raise
- self._family = family
- self._last_accepted = None
-
+ if maxsize is not None and size > maxsize:
+ return None
+ return self._recv(size)
+
+ def _poll(self, timeout):
+ r = wait([self], timeout)
+ return bool(r)
+
+
+#
+# Public functions
+#
+
+class Listener(object):
+ '''
+ Returns a listener object.
+
+ This is a wrapper for a bound socket which is 'listening' for
+ connections, or for a Windows named pipe.
+ '''
+ def __init__(self, address=None, family=None, backlog=1, authkey=None):
+ family = family or (address and address_type(address)) \
+ or default_family
+ address = address or arbitrary_address(family)
+
+ _validate_family(family)
+ if family == 'AF_PIPE':
+ self._listener = PipeListener(address, backlog)
+ else:
+ self._listener = SocketListener(address, family, backlog)
+
+ if authkey is not None and not isinstance(authkey, bytes):
+ raise TypeError('authkey should be a byte string')
+
+ self._authkey = authkey
+
+ def accept(self):
+ '''
+ Accept a connection on the bound socket or named pipe of `self`.
+
+ Returns a `Connection` object.
+ '''
+ if self._listener is None:
+ raise OSError('listener is closed')
+ c = self._listener.accept()
+ if self._authkey:
+ deliver_challenge(c, self._authkey)
+ answer_challenge(c, self._authkey)
+ return c
+
+ def close(self):
+ '''
+ Close the bound socket or named pipe of `self`.
+ '''
+ listener = self._listener
+ if listener is not None:
+ self._listener = None
+ listener.close()
+
+ @property
+ def address(self):
+ return self._listener._address
+
+ @property
+ def last_accepted(self):
+ return self._listener._last_accepted
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ self.close()
+
+
+def Client(address, family=None, authkey=None):
+ '''
+ Returns a connection to the address of a `Listener`
+ '''
+ family = family or address_type(address)
+ _validate_family(family)
+ if family == 'AF_PIPE':
+ c = PipeClient(address)
+ else:
+ c = SocketClient(address)
+
+ if authkey is not None and not isinstance(authkey, bytes):
+ raise TypeError('authkey should be a byte string')
+
+ if authkey is not None:
+ answer_challenge(c, authkey)
+ deliver_challenge(c, authkey)
+
+ return c
+
+
+if sys.platform != 'win32':
+
+ def Pipe(duplex=True):
+ '''
+ Returns pair of connection objects at either end of a pipe
+ '''
+ if duplex:
+ s1, s2 = socket.socketpair()
+ s1.setblocking(True)
+ s2.setblocking(True)
+ c1 = Connection(s1.detach())
+ c2 = Connection(s2.detach())
+ else:
+ fd1, fd2 = os.pipe()
+ c1 = Connection(fd1, writable=False)
+ c2 = Connection(fd2, readable=False)
+
+ return c1, c2
+
+else:
+
+ def Pipe(duplex=True):
+ '''
+ Returns pair of connection objects at either end of a pipe
+ '''
+ address = arbitrary_address('AF_PIPE')
+ if duplex:
+ openmode = _winapi.PIPE_ACCESS_DUPLEX
+ access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
+ obsize, ibsize = BUFSIZE, BUFSIZE
+ else:
+ openmode = _winapi.PIPE_ACCESS_INBOUND
+ access = _winapi.GENERIC_WRITE
+ obsize, ibsize = 0, BUFSIZE
+
+ h1 = _winapi.CreateNamedPipe(
+ address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
+ _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
+ _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
+ _winapi.PIPE_WAIT,
+ 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
+ # default security descriptor: the handle cannot be inherited
+ _winapi.NULL
+ )
+ h2 = _winapi.CreateFile(
+ address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
+ _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
+ )
+ _winapi.SetNamedPipeHandleState(
+ h2, _winapi.PIPE_READMODE_MESSAGE, None, None
+ )
+
+ overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
+ _, err = overlapped.GetOverlappedResult(True)
+ assert err == 0
+
+ c1 = PipeConnection(h1, writable=duplex)
+ c2 = PipeConnection(h2, readable=duplex)
+
+ return c1, c2
+
+#
+# Definitions for connections based on sockets
+#
+
+class SocketListener(object):
+ '''
+ Representation of a socket which is bound to an address and listening
+ '''
+ def __init__(self, address, family, backlog=1):
+ self._socket = socket.socket(getattr(socket, family))
+ try:
+ # SO_REUSEADDR has different semantics on Windows (issue #2550).
+ if os.name == 'posix':
+ self._socket.setsockopt(socket.SOL_SOCKET,
+ socket.SO_REUSEADDR, 1)
+ self._socket.setblocking(True)
+ self._socket.bind(address)
+ self._socket.listen(backlog)
+ self._address = self._socket.getsockname()
+ except OSError:
+ self._socket.close()
+ raise
+ self._family = family
+ self._last_accepted = None
+
if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
# Linux abstract socket namespaces do not need to be explicitly unlinked
- self._unlink = util.Finalize(
- self, os.unlink, args=(address,), exitpriority=0
- )
- else:
- self._unlink = None
-
- def accept(self):
- s, self._last_accepted = self._socket.accept()
- s.setblocking(True)
- return Connection(s.detach())
-
- def close(self):
- try:
- self._socket.close()
- finally:
- unlink = self._unlink
- if unlink is not None:
- self._unlink = None
- unlink()
-
-
-def SocketClient(address):
- '''
- Return a connection object connected to the socket given by `address`
- '''
- family = address_type(address)
- with socket.socket( getattr(socket, family) ) as s:
- s.setblocking(True)
- s.connect(address)
- return Connection(s.detach())
-
-#
-# Definitions for connections based on named pipes
-#
-
-if sys.platform == 'win32':
-
- class PipeListener(object):
- '''
- Representation of a named pipe
- '''
- def __init__(self, address, backlog=None):
- self._address = address
- self._handle_queue = [self._new_handle(first=True)]
-
- self._last_accepted = None
- util.sub_debug('listener created with address=%r', self._address)
- self.close = util.Finalize(
- self, PipeListener._finalize_pipe_listener,
- args=(self._handle_queue, self._address), exitpriority=0
- )
-
- def _new_handle(self, first=False):
- flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
- if first:
- flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
- return _winapi.CreateNamedPipe(
- self._address, flags,
- _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
- _winapi.PIPE_WAIT,
- _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
- _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
- )
-
- def accept(self):
- self._handle_queue.append(self._new_handle())
- handle = self._handle_queue.pop(0)
- try:
- ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
- except OSError as e:
- if e.winerror != _winapi.ERROR_NO_DATA:
- raise
- # ERROR_NO_DATA can occur if a client has already connected,
- # written data and then disconnected -- see Issue 14725.
- else:
- try:
- res = _winapi.WaitForMultipleObjects(
- [ov.event], False, INFINITE)
- except:
- ov.cancel()
- _winapi.CloseHandle(handle)
- raise
- finally:
- _, err = ov.GetOverlappedResult(True)
- assert err == 0
- return PipeConnection(handle)
-
- @staticmethod
- def _finalize_pipe_listener(queue, address):
- util.sub_debug('closing listener with address=%r', address)
- for handle in queue:
- _winapi.CloseHandle(handle)
-
- def PipeClient(address):
- '''
- Return a connection object connected to the pipe given by `address`
- '''
- t = _init_timeout()
- while 1:
- try:
- _winapi.WaitNamedPipe(address, 1000)
- h = _winapi.CreateFile(
- address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
- 0, _winapi.NULL, _winapi.OPEN_EXISTING,
- _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
- )
- except OSError as e:
- if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
- _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
- raise
- else:
- break
- else:
- raise
-
- _winapi.SetNamedPipeHandleState(
- h, _winapi.PIPE_READMODE_MESSAGE, None, None
- )
- return PipeConnection(h)
-
-#
-# Authentication stuff
-#
-
-MESSAGE_LENGTH = 20
-
-CHALLENGE = b'#CHALLENGE#'
-WELCOME = b'#WELCOME#'
-FAILURE = b'#FAILURE#'
-
-def deliver_challenge(connection, authkey):
- import hmac
- if not isinstance(authkey, bytes):
- raise ValueError(
- "Authkey must be bytes, not {0!s}".format(type(authkey)))
- message = os.urandom(MESSAGE_LENGTH)
- connection.send_bytes(CHALLENGE + message)
- digest = hmac.new(authkey, message, 'md5').digest()
- response = connection.recv_bytes(256) # reject large message
- if response == digest:
- connection.send_bytes(WELCOME)
- else:
- connection.send_bytes(FAILURE)
- raise AuthenticationError('digest received was wrong')
-
-def answer_challenge(connection, authkey):
- import hmac
- if not isinstance(authkey, bytes):
- raise ValueError(
- "Authkey must be bytes, not {0!s}".format(type(authkey)))
- message = connection.recv_bytes(256) # reject large message
- assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
- message = message[len(CHALLENGE):]
- digest = hmac.new(authkey, message, 'md5').digest()
- connection.send_bytes(digest)
- response = connection.recv_bytes(256) # reject large message
- if response != WELCOME:
- raise AuthenticationError('digest sent was rejected')
-
-#
-# Support for using xmlrpclib for serialization
-#
-
-class ConnectionWrapper(object):
- def __init__(self, conn, dumps, loads):
- self._conn = conn
- self._dumps = dumps
- self._loads = loads
- for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
- obj = getattr(conn, attr)
- setattr(self, attr, obj)
- def send(self, obj):
- s = self._dumps(obj)
- self._conn.send_bytes(s)
- def recv(self):
- s = self._conn.recv_bytes()
- return self._loads(s)
-
-def _xml_dumps(obj):
- return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
-
-def _xml_loads(s):
- (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
- return obj
-
-class XmlListener(Listener):
- def accept(self):
- global xmlrpclib
- import xmlrpc.client as xmlrpclib
- obj = Listener.accept(self)
- return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
-
-def XmlClient(*args, **kwds):
- global xmlrpclib
- import xmlrpc.client as xmlrpclib
- return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
-
-#
-# Wait
-#
-
-if sys.platform == 'win32':
-
- def _exhaustive_wait(handles, timeout):
- # Return ALL handles which are currently signalled. (Only
- # returning the first signalled might create starvation issues.)
- L = list(handles)
- ready = []
- while L:
- res = _winapi.WaitForMultipleObjects(L, False, timeout)
- if res == WAIT_TIMEOUT:
- break
- elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
- res -= WAIT_OBJECT_0
- elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
- res -= WAIT_ABANDONED_0
- else:
- raise RuntimeError('Should not get here')
- ready.append(L[res])
- L = L[res+1:]
- timeout = 0
- return ready
-
- _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
-
- def wait(object_list, timeout=None):
- '''
- Wait till an object in object_list is ready/readable.
-
- Returns list of those objects in object_list which are ready/readable.
- '''
- if timeout is None:
- timeout = INFINITE
- elif timeout < 0:
- timeout = 0
- else:
- timeout = int(timeout * 1000 + 0.5)
-
- object_list = list(object_list)
- waithandle_to_obj = {}
- ov_list = []
- ready_objects = set()
- ready_handles = set()
-
- try:
- for o in object_list:
- try:
- fileno = getattr(o, 'fileno')
- except AttributeError:
- waithandle_to_obj[o.__index__()] = o
- else:
- # start an overlapped read of length zero
- try:
- ov, err = _winapi.ReadFile(fileno(), 0, True)
- except OSError as e:
- ov, err = None, e.winerror
- if err not in _ready_errors:
- raise
- if err == _winapi.ERROR_IO_PENDING:
- ov_list.append(ov)
- waithandle_to_obj[ov.event] = o
- else:
- # If o.fileno() is an overlapped pipe handle and
- # err == 0 then there is a zero length message
- # in the pipe, but it HAS NOT been consumed...
- if ov and sys.getwindowsversion()[:2] >= (6, 2):
- # ... except on Windows 8 and later, where
- # the message HAS been consumed.
- try:
- _, err = ov.GetOverlappedResult(False)
- except OSError as e:
- err = e.winerror
- if not err and hasattr(o, '_got_empty_message'):
- o._got_empty_message = True
- ready_objects.add(o)
- timeout = 0
-
- ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
- finally:
- # request that overlapped reads stop
- for ov in ov_list:
- ov.cancel()
-
- # wait for all overlapped reads to stop
- for ov in ov_list:
- try:
- _, err = ov.GetOverlappedResult(True)
- except OSError as e:
- err = e.winerror
- if err not in _ready_errors:
- raise
- if err != _winapi.ERROR_OPERATION_ABORTED:
- o = waithandle_to_obj[ov.event]
- ready_objects.add(o)
- if err == 0:
- # If o.fileno() is an overlapped pipe handle then
- # a zero length message HAS been consumed.
- if hasattr(o, '_got_empty_message'):
- o._got_empty_message = True
-
- ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
- return [o for o in object_list if o in ready_objects]
-
-else:
-
- import selectors
-
- # poll/select have the advantage of not requiring any extra file
- # descriptor, contrarily to epoll/kqueue (also, they require a single
- # syscall).
- if hasattr(selectors, 'PollSelector'):
- _WaitSelector = selectors.PollSelector
- else:
- _WaitSelector = selectors.SelectSelector
-
- def wait(object_list, timeout=None):
- '''
- Wait till an object in object_list is ready/readable.
-
- Returns list of those objects in object_list which are ready/readable.
- '''
- with _WaitSelector() as selector:
- for obj in object_list:
- selector.register(obj, selectors.EVENT_READ)
-
- if timeout is not None:
- deadline = time.monotonic() + timeout
-
- while True:
- ready = selector.select(timeout)
- if ready:
- return [key.fileobj for (key, events) in ready]
- else:
- if timeout is not None:
- timeout = deadline - time.monotonic()
- if timeout < 0:
- return ready
-
-#
-# Make connection and socket objects sharable if possible
-#
-
-if sys.platform == 'win32':
- def reduce_connection(conn):
- handle = conn.fileno()
- with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
- from . import resource_sharer
- ds = resource_sharer.DupSocket(s)
- return rebuild_connection, (ds, conn.readable, conn.writable)
- def rebuild_connection(ds, readable, writable):
- sock = ds.detach()
- return Connection(sock.detach(), readable, writable)
- reduction.register(Connection, reduce_connection)
-
- def reduce_pipe_connection(conn):
- access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
- (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
- dh = reduction.DupHandle(conn.fileno(), access)
- return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
- def rebuild_pipe_connection(dh, readable, writable):
- handle = dh.detach()
- return PipeConnection(handle, readable, writable)
- reduction.register(PipeConnection, reduce_pipe_connection)
-
-else:
- def reduce_connection(conn):
- df = reduction.DupFd(conn.fileno())
- return rebuild_connection, (df, conn.readable, conn.writable)
- def rebuild_connection(df, readable, writable):
- fd = df.detach()
- return Connection(fd, readable, writable)
- reduction.register(Connection, reduce_connection)
+ self._unlink = util.Finalize(
+ self, os.unlink, args=(address,), exitpriority=0
+ )
+ else:
+ self._unlink = None
+
+ def accept(self):
+ s, self._last_accepted = self._socket.accept()
+ s.setblocking(True)
+ return Connection(s.detach())
+
+ def close(self):
+ try:
+ self._socket.close()
+ finally:
+ unlink = self._unlink
+ if unlink is not None:
+ self._unlink = None
+ unlink()
+
+
+def SocketClient(address):
+ '''
+ Return a connection object connected to the socket given by `address`
+ '''
+ family = address_type(address)
+ with socket.socket( getattr(socket, family) ) as s:
+ s.setblocking(True)
+ s.connect(address)
+ return Connection(s.detach())
+
+#
+# Definitions for connections based on named pipes
+#
+
+if sys.platform == 'win32':
+
+ class PipeListener(object):
+ '''
+ Representation of a named pipe
+ '''
+ def __init__(self, address, backlog=None):
+ self._address = address
+ self._handle_queue = [self._new_handle(first=True)]
+
+ self._last_accepted = None
+ util.sub_debug('listener created with address=%r', self._address)
+ self.close = util.Finalize(
+ self, PipeListener._finalize_pipe_listener,
+ args=(self._handle_queue, self._address), exitpriority=0
+ )
+
+ def _new_handle(self, first=False):
+ flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
+ if first:
+ flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
+ return _winapi.CreateNamedPipe(
+ self._address, flags,
+ _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
+ _winapi.PIPE_WAIT,
+ _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
+ _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
+ )
+
+ def accept(self):
+ self._handle_queue.append(self._new_handle())
+ handle = self._handle_queue.pop(0)
+ try:
+ ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
+ except OSError as e:
+ if e.winerror != _winapi.ERROR_NO_DATA:
+ raise
+ # ERROR_NO_DATA can occur if a client has already connected,
+ # written data and then disconnected -- see Issue 14725.
+ else:
+ try:
+ res = _winapi.WaitForMultipleObjects(
+ [ov.event], False, INFINITE)
+ except:
+ ov.cancel()
+ _winapi.CloseHandle(handle)
+ raise
+ finally:
+ _, err = ov.GetOverlappedResult(True)
+ assert err == 0
+ return PipeConnection(handle)
+
+ @staticmethod
+ def _finalize_pipe_listener(queue, address):
+ util.sub_debug('closing listener with address=%r', address)
+ for handle in queue:
+ _winapi.CloseHandle(handle)
+
+ def PipeClient(address):
+ '''
+ Return a connection object connected to the pipe given by `address`
+ '''
+ t = _init_timeout()
+ while 1:
+ try:
+ _winapi.WaitNamedPipe(address, 1000)
+ h = _winapi.CreateFile(
+ address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
+ 0, _winapi.NULL, _winapi.OPEN_EXISTING,
+ _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
+ )
+ except OSError as e:
+ if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
+ _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
+ raise
+ else:
+ break
+ else:
+ raise
+
+ _winapi.SetNamedPipeHandleState(
+ h, _winapi.PIPE_READMODE_MESSAGE, None, None
+ )
+ return PipeConnection(h)
+
+#
+# Authentication stuff
+#
+
+MESSAGE_LENGTH = 20
+
+CHALLENGE = b'#CHALLENGE#'
+WELCOME = b'#WELCOME#'
+FAILURE = b'#FAILURE#'
+
+def deliver_challenge(connection, authkey):
+ import hmac
+ if not isinstance(authkey, bytes):
+ raise ValueError(
+ "Authkey must be bytes, not {0!s}".format(type(authkey)))
+ message = os.urandom(MESSAGE_LENGTH)
+ connection.send_bytes(CHALLENGE + message)
+ digest = hmac.new(authkey, message, 'md5').digest()
+ response = connection.recv_bytes(256) # reject large message
+ if response == digest:
+ connection.send_bytes(WELCOME)
+ else:
+ connection.send_bytes(FAILURE)
+ raise AuthenticationError('digest received was wrong')
+
+def answer_challenge(connection, authkey):
+ import hmac
+ if not isinstance(authkey, bytes):
+ raise ValueError(
+ "Authkey must be bytes, not {0!s}".format(type(authkey)))
+ message = connection.recv_bytes(256) # reject large message
+ assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
+ message = message[len(CHALLENGE):]
+ digest = hmac.new(authkey, message, 'md5').digest()
+ connection.send_bytes(digest)
+ response = connection.recv_bytes(256) # reject large message
+ if response != WELCOME:
+ raise AuthenticationError('digest sent was rejected')
+
+#
+# Support for using xmlrpclib for serialization
+#
+
+class ConnectionWrapper(object):
+ def __init__(self, conn, dumps, loads):
+ self._conn = conn
+ self._dumps = dumps
+ self._loads = loads
+ for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
+ obj = getattr(conn, attr)
+ setattr(self, attr, obj)
+ def send(self, obj):
+ s = self._dumps(obj)
+ self._conn.send_bytes(s)
+ def recv(self):
+ s = self._conn.recv_bytes()
+ return self._loads(s)
+
+def _xml_dumps(obj):
+ return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
+
+def _xml_loads(s):
+ (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
+ return obj
+
+class XmlListener(Listener):
+ def accept(self):
+ global xmlrpclib
+ import xmlrpc.client as xmlrpclib
+ obj = Listener.accept(self)
+ return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
+
+def XmlClient(*args, **kwds):
+ global xmlrpclib
+ import xmlrpc.client as xmlrpclib
+ return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
+
+#
+# Wait
+#
+
+if sys.platform == 'win32':
+
+ def _exhaustive_wait(handles, timeout):
+ # Return ALL handles which are currently signalled. (Only
+ # returning the first signalled might create starvation issues.)
+ L = list(handles)
+ ready = []
+ while L:
+ res = _winapi.WaitForMultipleObjects(L, False, timeout)
+ if res == WAIT_TIMEOUT:
+ break
+ elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
+ res -= WAIT_OBJECT_0
+ elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
+ res -= WAIT_ABANDONED_0
+ else:
+ raise RuntimeError('Should not get here')
+ ready.append(L[res])
+ L = L[res+1:]
+ timeout = 0
+ return ready
+
+ _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
+
+ def wait(object_list, timeout=None):
+ '''
+ Wait till an object in object_list is ready/readable.
+
+ Returns list of those objects in object_list which are ready/readable.
+ '''
+ if timeout is None:
+ timeout = INFINITE
+ elif timeout < 0:
+ timeout = 0
+ else:
+ timeout = int(timeout * 1000 + 0.5)
+
+ object_list = list(object_list)
+ waithandle_to_obj = {}
+ ov_list = []
+ ready_objects = set()
+ ready_handles = set()
+
+ try:
+ for o in object_list:
+ try:
+ fileno = getattr(o, 'fileno')
+ except AttributeError:
+ waithandle_to_obj[o.__index__()] = o
+ else:
+ # start an overlapped read of length zero
+ try:
+ ov, err = _winapi.ReadFile(fileno(), 0, True)
+ except OSError as e:
+ ov, err = None, e.winerror
+ if err not in _ready_errors:
+ raise
+ if err == _winapi.ERROR_IO_PENDING:
+ ov_list.append(ov)
+ waithandle_to_obj[ov.event] = o
+ else:
+ # If o.fileno() is an overlapped pipe handle and
+ # err == 0 then there is a zero length message
+ # in the pipe, but it HAS NOT been consumed...
+ if ov and sys.getwindowsversion()[:2] >= (6, 2):
+ # ... except on Windows 8 and later, where
+ # the message HAS been consumed.
+ try:
+ _, err = ov.GetOverlappedResult(False)
+ except OSError as e:
+ err = e.winerror
+ if not err and hasattr(o, '_got_empty_message'):
+ o._got_empty_message = True
+ ready_objects.add(o)
+ timeout = 0
+
+ ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
+ finally:
+ # request that overlapped reads stop
+ for ov in ov_list:
+ ov.cancel()
+
+ # wait for all overlapped reads to stop
+ for ov in ov_list:
+ try:
+ _, err = ov.GetOverlappedResult(True)
+ except OSError as e:
+ err = e.winerror
+ if err not in _ready_errors:
+ raise
+ if err != _winapi.ERROR_OPERATION_ABORTED:
+ o = waithandle_to_obj[ov.event]
+ ready_objects.add(o)
+ if err == 0:
+ # If o.fileno() is an overlapped pipe handle then
+ # a zero length message HAS been consumed.
+ if hasattr(o, '_got_empty_message'):
+ o._got_empty_message = True
+
+ ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
+ return [o for o in object_list if o in ready_objects]
+
+else:
+
+ import selectors
+
+ # poll/select have the advantage of not requiring any extra file
+ # descriptor, contrarily to epoll/kqueue (also, they require a single
+ # syscall).
+ if hasattr(selectors, 'PollSelector'):
+ _WaitSelector = selectors.PollSelector
+ else:
+ _WaitSelector = selectors.SelectSelector
+
+ def wait(object_list, timeout=None):
+ '''
+ Wait till an object in object_list is ready/readable.
+
+ Returns list of those objects in object_list which are ready/readable.
+ '''
+ with _WaitSelector() as selector:
+ for obj in object_list:
+ selector.register(obj, selectors.EVENT_READ)
+
+ if timeout is not None:
+ deadline = time.monotonic() + timeout
+
+ while True:
+ ready = selector.select(timeout)
+ if ready:
+ return [key.fileobj for (key, events) in ready]
+ else:
+ if timeout is not None:
+ timeout = deadline - time.monotonic()
+ if timeout < 0:
+ return ready
+
+#
+# Make connection and socket objects sharable if possible
+#
+
+if sys.platform == 'win32':
+ def reduce_connection(conn):
+ handle = conn.fileno()
+ with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
+ from . import resource_sharer
+ ds = resource_sharer.DupSocket(s)
+ return rebuild_connection, (ds, conn.readable, conn.writable)
+ def rebuild_connection(ds, readable, writable):
+ sock = ds.detach()
+ return Connection(sock.detach(), readable, writable)
+ reduction.register(Connection, reduce_connection)
+
+ def reduce_pipe_connection(conn):
+ access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
+ (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
+ dh = reduction.DupHandle(conn.fileno(), access)
+ return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
+ def rebuild_pipe_connection(dh, readable, writable):
+ handle = dh.detach()
+ return PipeConnection(handle, readable, writable)
+ reduction.register(PipeConnection, reduce_pipe_connection)
+
+else:
+ def reduce_connection(conn):
+ df = reduction.DupFd(conn.fileno())
+ return rebuild_connection, (df, conn.readable, conn.writable)
+ def rebuild_connection(df, readable, writable):
+ fd = df.detach()
+ return Connection(fd, readable, writable)
+ reduction.register(Connection, reduce_connection)
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/context.py b/contrib/tools/python3/src/Lib/multiprocessing/context.py
index 8d0525d5d62..d2f41241dc2 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/context.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/context.py
@@ -1,362 +1,362 @@
-import os
-import sys
-import threading
-
-from . import process
-from . import reduction
-
+import os
+import sys
+import threading
+
+from . import process
+from . import reduction
+
__all__ = ()
-
-#
-# Exceptions
-#
-
-class ProcessError(Exception):
- pass
-
-class BufferTooShort(ProcessError):
- pass
-
-class TimeoutError(ProcessError):
- pass
-
-class AuthenticationError(ProcessError):
- pass
-
-#
+
+#
+# Exceptions
+#
+
+class ProcessError(Exception):
+ pass
+
+class BufferTooShort(ProcessError):
+ pass
+
+class TimeoutError(ProcessError):
+ pass
+
+class AuthenticationError(ProcessError):
+ pass
+
+#
# Base type for contexts. Bound methods of an instance of this type are included in __all__ of __init__.py
-#
-
-class BaseContext(object):
-
- ProcessError = ProcessError
- BufferTooShort = BufferTooShort
- TimeoutError = TimeoutError
- AuthenticationError = AuthenticationError
-
- current_process = staticmethod(process.current_process)
+#
+
+class BaseContext(object):
+
+ ProcessError = ProcessError
+ BufferTooShort = BufferTooShort
+ TimeoutError = TimeoutError
+ AuthenticationError = AuthenticationError
+
+ current_process = staticmethod(process.current_process)
parent_process = staticmethod(process.parent_process)
- active_children = staticmethod(process.active_children)
-
- def cpu_count(self):
- '''Returns the number of CPUs in the system'''
- num = os.cpu_count()
- if num is None:
- raise NotImplementedError('cannot determine number of cpus')
- else:
- return num
-
- def Manager(self):
- '''Returns a manager associated with a running server process
-
- The managers methods such as `Lock()`, `Condition()` and `Queue()`
- can be used to create shared objects.
- '''
- from .managers import SyncManager
- m = SyncManager(ctx=self.get_context())
- m.start()
- return m
-
- def Pipe(self, duplex=True):
- '''Returns two connection object connected by a pipe'''
- from .connection import Pipe
- return Pipe(duplex)
-
- def Lock(self):
- '''Returns a non-recursive lock object'''
- from .synchronize import Lock
- return Lock(ctx=self.get_context())
-
- def RLock(self):
- '''Returns a recursive lock object'''
- from .synchronize import RLock
- return RLock(ctx=self.get_context())
-
- def Condition(self, lock=None):
- '''Returns a condition object'''
- from .synchronize import Condition
- return Condition(lock, ctx=self.get_context())
-
- def Semaphore(self, value=1):
- '''Returns a semaphore object'''
- from .synchronize import Semaphore
- return Semaphore(value, ctx=self.get_context())
-
- def BoundedSemaphore(self, value=1):
- '''Returns a bounded semaphore object'''
- from .synchronize import BoundedSemaphore
- return BoundedSemaphore(value, ctx=self.get_context())
-
- def Event(self):
- '''Returns an event object'''
- from .synchronize import Event
- return Event(ctx=self.get_context())
-
- def Barrier(self, parties, action=None, timeout=None):
- '''Returns a barrier object'''
- from .synchronize import Barrier
- return Barrier(parties, action, timeout, ctx=self.get_context())
-
- def Queue(self, maxsize=0):
- '''Returns a queue object'''
- from .queues import Queue
- return Queue(maxsize, ctx=self.get_context())
-
- def JoinableQueue(self, maxsize=0):
- '''Returns a queue object'''
- from .queues import JoinableQueue
- return JoinableQueue(maxsize, ctx=self.get_context())
-
- def SimpleQueue(self):
- '''Returns a queue object'''
- from .queues import SimpleQueue
- return SimpleQueue(ctx=self.get_context())
-
- def Pool(self, processes=None, initializer=None, initargs=(),
- maxtasksperchild=None):
- '''Returns a process pool object'''
- from .pool import Pool
- return Pool(processes, initializer, initargs, maxtasksperchild,
- context=self.get_context())
-
- def RawValue(self, typecode_or_type, *args):
- '''Returns a shared object'''
- from .sharedctypes import RawValue
- return RawValue(typecode_or_type, *args)
-
- def RawArray(self, typecode_or_type, size_or_initializer):
- '''Returns a shared array'''
- from .sharedctypes import RawArray
- return RawArray(typecode_or_type, size_or_initializer)
-
- def Value(self, typecode_or_type, *args, lock=True):
- '''Returns a synchronized shared object'''
- from .sharedctypes import Value
- return Value(typecode_or_type, *args, lock=lock,
- ctx=self.get_context())
-
- def Array(self, typecode_or_type, size_or_initializer, *, lock=True):
- '''Returns a synchronized shared array'''
- from .sharedctypes import Array
- return Array(typecode_or_type, size_or_initializer, lock=lock,
- ctx=self.get_context())
-
- def freeze_support(self):
- '''Check whether this is a fake forked process in a frozen executable.
- If so then run code specified by commandline and exit.
- '''
- if sys.platform == 'win32' and getattr(sys, 'frozen', False):
- from .spawn import freeze_support
- freeze_support()
-
- def get_logger(self):
- '''Return package logger -- if it does not already exist then
- it is created.
- '''
- from .util import get_logger
- return get_logger()
-
- def log_to_stderr(self, level=None):
- '''Turn on logging and add a handler which prints to stderr'''
- from .util import log_to_stderr
- return log_to_stderr(level)
-
- def allow_connection_pickling(self):
- '''Install support for sending connections and sockets
- between processes
- '''
- # This is undocumented. In previous versions of multiprocessing
- # its only effect was to make socket objects inheritable on Windows.
- from . import connection
-
- def set_executable(self, executable):
- '''Sets the path to a python.exe or pythonw.exe binary used to run
- child processes instead of sys.executable when using the 'spawn'
- start method. Useful for people embedding Python.
- '''
- from .spawn import set_executable
- set_executable(executable)
-
- def set_forkserver_preload(self, module_names):
- '''Set list of module names to try to load in forkserver process.
- This is really just a hint.
- '''
- from .forkserver import set_forkserver_preload
- set_forkserver_preload(module_names)
-
- def get_context(self, method=None):
- if method is None:
- return self
- try:
- ctx = _concrete_contexts[method]
- except KeyError:
- raise ValueError('cannot find context for %r' % method) from None
- ctx._check_available()
- return ctx
-
- def get_start_method(self, allow_none=False):
- return self._name
-
- def set_start_method(self, method, force=False):
- raise ValueError('cannot set start method of concrete context')
-
- @property
- def reducer(self):
- '''Controls how objects will be reduced to a form that can be
- shared with other processes.'''
- return globals().get('reduction')
-
- @reducer.setter
- def reducer(self, reduction):
- globals()['reduction'] = reduction
-
- def _check_available(self):
- pass
-
-#
-# Type of default context -- underlying context can be set at most once
-#
-
-class Process(process.BaseProcess):
- _start_method = None
- @staticmethod
- def _Popen(process_obj):
- return _default_context.get_context().Process._Popen(process_obj)
-
-class DefaultContext(BaseContext):
- Process = Process
-
- def __init__(self, context):
- self._default_context = context
- self._actual_context = None
-
- def get_context(self, method=None):
- if method is None:
- if self._actual_context is None:
- self._actual_context = self._default_context
- return self._actual_context
- else:
- return super().get_context(method)
-
- def set_start_method(self, method, force=False):
- if self._actual_context is not None and not force:
- raise RuntimeError('context has already been set')
- if method is None and force:
- self._actual_context = None
- return
- self._actual_context = self.get_context(method)
-
- def get_start_method(self, allow_none=False):
- if self._actual_context is None:
- if allow_none:
- return None
- self._actual_context = self._default_context
- return self._actual_context._name
-
- def get_all_start_methods(self):
- if sys.platform == 'win32':
- return ['spawn']
- else:
+ active_children = staticmethod(process.active_children)
+
+ def cpu_count(self):
+ '''Returns the number of CPUs in the system'''
+ num = os.cpu_count()
+ if num is None:
+ raise NotImplementedError('cannot determine number of cpus')
+ else:
+ return num
+
+ def Manager(self):
+ '''Returns a manager associated with a running server process
+
+ The managers methods such as `Lock()`, `Condition()` and `Queue()`
+ can be used to create shared objects.
+ '''
+ from .managers import SyncManager
+ m = SyncManager(ctx=self.get_context())
+ m.start()
+ return m
+
+ def Pipe(self, duplex=True):
+ '''Returns two connection object connected by a pipe'''
+ from .connection import Pipe
+ return Pipe(duplex)
+
+ def Lock(self):
+ '''Returns a non-recursive lock object'''
+ from .synchronize import Lock
+ return Lock(ctx=self.get_context())
+
+ def RLock(self):
+ '''Returns a recursive lock object'''
+ from .synchronize import RLock
+ return RLock(ctx=self.get_context())
+
+ def Condition(self, lock=None):
+ '''Returns a condition object'''
+ from .synchronize import Condition
+ return Condition(lock, ctx=self.get_context())
+
+ def Semaphore(self, value=1):
+ '''Returns a semaphore object'''
+ from .synchronize import Semaphore
+ return Semaphore(value, ctx=self.get_context())
+
+ def BoundedSemaphore(self, value=1):
+ '''Returns a bounded semaphore object'''
+ from .synchronize import BoundedSemaphore
+ return BoundedSemaphore(value, ctx=self.get_context())
+
+ def Event(self):
+ '''Returns an event object'''
+ from .synchronize import Event
+ return Event(ctx=self.get_context())
+
+ def Barrier(self, parties, action=None, timeout=None):
+ '''Returns a barrier object'''
+ from .synchronize import Barrier
+ return Barrier(parties, action, timeout, ctx=self.get_context())
+
+ def Queue(self, maxsize=0):
+ '''Returns a queue object'''
+ from .queues import Queue
+ return Queue(maxsize, ctx=self.get_context())
+
+ def JoinableQueue(self, maxsize=0):
+ '''Returns a queue object'''
+ from .queues import JoinableQueue
+ return JoinableQueue(maxsize, ctx=self.get_context())
+
+ def SimpleQueue(self):
+ '''Returns a queue object'''
+ from .queues import SimpleQueue
+ return SimpleQueue(ctx=self.get_context())
+
+ def Pool(self, processes=None, initializer=None, initargs=(),
+ maxtasksperchild=None):
+ '''Returns a process pool object'''
+ from .pool import Pool
+ return Pool(processes, initializer, initargs, maxtasksperchild,
+ context=self.get_context())
+
+ def RawValue(self, typecode_or_type, *args):
+ '''Returns a shared object'''
+ from .sharedctypes import RawValue
+ return RawValue(typecode_or_type, *args)
+
+ def RawArray(self, typecode_or_type, size_or_initializer):
+ '''Returns a shared array'''
+ from .sharedctypes import RawArray
+ return RawArray(typecode_or_type, size_or_initializer)
+
+ def Value(self, typecode_or_type, *args, lock=True):
+ '''Returns a synchronized shared object'''
+ from .sharedctypes import Value
+ return Value(typecode_or_type, *args, lock=lock,
+ ctx=self.get_context())
+
+ def Array(self, typecode_or_type, size_or_initializer, *, lock=True):
+ '''Returns a synchronized shared array'''
+ from .sharedctypes import Array
+ return Array(typecode_or_type, size_or_initializer, lock=lock,
+ ctx=self.get_context())
+
+ def freeze_support(self):
+ '''Check whether this is a fake forked process in a frozen executable.
+ If so then run code specified by commandline and exit.
+ '''
+ if sys.platform == 'win32' and getattr(sys, 'frozen', False):
+ from .spawn import freeze_support
+ freeze_support()
+
+ def get_logger(self):
+ '''Return package logger -- if it does not already exist then
+ it is created.
+ '''
+ from .util import get_logger
+ return get_logger()
+
+ def log_to_stderr(self, level=None):
+ '''Turn on logging and add a handler which prints to stderr'''
+ from .util import log_to_stderr
+ return log_to_stderr(level)
+
+ def allow_connection_pickling(self):
+ '''Install support for sending connections and sockets
+ between processes
+ '''
+ # This is undocumented. In previous versions of multiprocessing
+ # its only effect was to make socket objects inheritable on Windows.
+ from . import connection
+
+ def set_executable(self, executable):
+ '''Sets the path to a python.exe or pythonw.exe binary used to run
+ child processes instead of sys.executable when using the 'spawn'
+ start method. Useful for people embedding Python.
+ '''
+ from .spawn import set_executable
+ set_executable(executable)
+
+ def set_forkserver_preload(self, module_names):
+ '''Set list of module names to try to load in forkserver process.
+ This is really just a hint.
+ '''
+ from .forkserver import set_forkserver_preload
+ set_forkserver_preload(module_names)
+
+ def get_context(self, method=None):
+ if method is None:
+ return self
+ try:
+ ctx = _concrete_contexts[method]
+ except KeyError:
+ raise ValueError('cannot find context for %r' % method) from None
+ ctx._check_available()
+ return ctx
+
+ def get_start_method(self, allow_none=False):
+ return self._name
+
+ def set_start_method(self, method, force=False):
+ raise ValueError('cannot set start method of concrete context')
+
+ @property
+ def reducer(self):
+ '''Controls how objects will be reduced to a form that can be
+ shared with other processes.'''
+ return globals().get('reduction')
+
+ @reducer.setter
+ def reducer(self, reduction):
+ globals()['reduction'] = reduction
+
+ def _check_available(self):
+ pass
+
+#
+# Type of default context -- underlying context can be set at most once
+#
+
+class Process(process.BaseProcess):
+ _start_method = None
+ @staticmethod
+ def _Popen(process_obj):
+ return _default_context.get_context().Process._Popen(process_obj)
+
+class DefaultContext(BaseContext):
+ Process = Process
+
+ def __init__(self, context):
+ self._default_context = context
+ self._actual_context = None
+
+ def get_context(self, method=None):
+ if method is None:
+ if self._actual_context is None:
+ self._actual_context = self._default_context
+ return self._actual_context
+ else:
+ return super().get_context(method)
+
+ def set_start_method(self, method, force=False):
+ if self._actual_context is not None and not force:
+ raise RuntimeError('context has already been set')
+ if method is None and force:
+ self._actual_context = None
+ return
+ self._actual_context = self.get_context(method)
+
+ def get_start_method(self, allow_none=False):
+ if self._actual_context is None:
+ if allow_none:
+ return None
+ self._actual_context = self._default_context
+ return self._actual_context._name
+
+ def get_all_start_methods(self):
+ if sys.platform == 'win32':
+ return ['spawn']
+ else:
methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn']
- if reduction.HAVE_SEND_HANDLE:
+ if reduction.HAVE_SEND_HANDLE:
methods.append('forkserver')
return methods
-
-
-#
-# Context types for fixed start method
-#
-
-if sys.platform != 'win32':
-
- class ForkProcess(process.BaseProcess):
- _start_method = 'fork'
- @staticmethod
- def _Popen(process_obj):
- from .popen_fork import Popen
- return Popen(process_obj)
-
- class SpawnProcess(process.BaseProcess):
- _start_method = 'spawn'
- @staticmethod
- def _Popen(process_obj):
- from .popen_spawn_posix import Popen
- return Popen(process_obj)
-
- class ForkServerProcess(process.BaseProcess):
- _start_method = 'forkserver'
- @staticmethod
- def _Popen(process_obj):
- from .popen_forkserver import Popen
- return Popen(process_obj)
-
- class ForkContext(BaseContext):
- _name = 'fork'
- Process = ForkProcess
-
- class SpawnContext(BaseContext):
- _name = 'spawn'
- Process = SpawnProcess
-
- class ForkServerContext(BaseContext):
- _name = 'forkserver'
- Process = ForkServerProcess
- def _check_available(self):
- if not reduction.HAVE_SEND_HANDLE:
- raise ValueError('forkserver start method not available')
-
- _concrete_contexts = {
- 'fork': ForkContext(),
- 'spawn': SpawnContext(),
- 'forkserver': ForkServerContext(),
- }
+
+
+#
+# Context types for fixed start method
+#
+
+if sys.platform != 'win32':
+
+ class ForkProcess(process.BaseProcess):
+ _start_method = 'fork'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_fork import Popen
+ return Popen(process_obj)
+
+ class SpawnProcess(process.BaseProcess):
+ _start_method = 'spawn'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_spawn_posix import Popen
+ return Popen(process_obj)
+
+ class ForkServerProcess(process.BaseProcess):
+ _start_method = 'forkserver'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_forkserver import Popen
+ return Popen(process_obj)
+
+ class ForkContext(BaseContext):
+ _name = 'fork'
+ Process = ForkProcess
+
+ class SpawnContext(BaseContext):
+ _name = 'spawn'
+ Process = SpawnProcess
+
+ class ForkServerContext(BaseContext):
+ _name = 'forkserver'
+ Process = ForkServerProcess
+ def _check_available(self):
+ if not reduction.HAVE_SEND_HANDLE:
+ raise ValueError('forkserver start method not available')
+
+ _concrete_contexts = {
+ 'fork': ForkContext(),
+ 'spawn': SpawnContext(),
+ 'forkserver': ForkServerContext(),
+ }
if sys.platform == 'darwin':
# bpo-33725: running arbitrary code after fork() is no longer reliable
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
_default_context = DefaultContext(_concrete_contexts['spawn'])
else:
_default_context = DefaultContext(_concrete_contexts['fork'])
-
-else:
-
- class SpawnProcess(process.BaseProcess):
- _start_method = 'spawn'
- @staticmethod
- def _Popen(process_obj):
- from .popen_spawn_win32 import Popen
- return Popen(process_obj)
-
- class SpawnContext(BaseContext):
- _name = 'spawn'
- Process = SpawnProcess
-
- _concrete_contexts = {
- 'spawn': SpawnContext(),
- }
- _default_context = DefaultContext(_concrete_contexts['spawn'])
-
-#
-# Force the start method
-#
-
-def _force_start_method(method):
- _default_context._actual_context = _concrete_contexts[method]
-
-#
-# Check that the current thread is spawning a child process
-#
-
-_tls = threading.local()
-
-def get_spawning_popen():
- return getattr(_tls, 'spawning_popen', None)
-
-def set_spawning_popen(popen):
- _tls.spawning_popen = popen
-
-def assert_spawning(obj):
- if get_spawning_popen() is None:
- raise RuntimeError(
- '%s objects should only be shared between processes'
- ' through inheritance' % type(obj).__name__
- )
+
+else:
+
+ class SpawnProcess(process.BaseProcess):
+ _start_method = 'spawn'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_spawn_win32 import Popen
+ return Popen(process_obj)
+
+ class SpawnContext(BaseContext):
+ _name = 'spawn'
+ Process = SpawnProcess
+
+ _concrete_contexts = {
+ 'spawn': SpawnContext(),
+ }
+ _default_context = DefaultContext(_concrete_contexts['spawn'])
+
+#
+# Force the start method
+#
+
+def _force_start_method(method):
+ _default_context._actual_context = _concrete_contexts[method]
+
+#
+# Check that the current thread is spawning a child process
+#
+
+_tls = threading.local()
+
+def get_spawning_popen():
+ return getattr(_tls, 'spawning_popen', None)
+
+def set_spawning_popen(popen):
+ _tls.spawning_popen = popen
+
+def assert_spawning(obj):
+ if get_spawning_popen() is None:
+ raise RuntimeError(
+ '%s objects should only be shared between processes'
+ ' through inheritance' % type(obj).__name__
+ )
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/dummy/__init__.py b/contrib/tools/python3/src/Lib/multiprocessing/dummy/__init__.py
index 6a1468609e3..9705e818711 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/dummy/__init__.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/dummy/__init__.py
@@ -1,126 +1,126 @@
-#
-# Support for the API of the multiprocessing package using threads
-#
-# multiprocessing/dummy/__init__.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-__all__ = [
- 'Process', 'current_process', 'active_children', 'freeze_support',
- 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
- 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue'
- ]
-
-#
-# Imports
-#
-
-import threading
-import sys
-import weakref
-import array
-
-from .connection import Pipe
-from threading import Lock, RLock, Semaphore, BoundedSemaphore
-from threading import Event, Condition, Barrier
-from queue import Queue
-
-#
-#
-#
-
-class DummyProcess(threading.Thread):
-
- def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
- threading.Thread.__init__(self, group, target, name, args, kwargs)
- self._pid = None
- self._children = weakref.WeakKeyDictionary()
- self._start_called = False
- self._parent = current_process()
-
- def start(self):
- if self._parent is not current_process():
- raise RuntimeError(
- "Parent is {0!r} but current_process is {1!r}".format(
- self._parent, current_process()))
- self._start_called = True
- if hasattr(self._parent, '_children'):
- self._parent._children[self] = None
- threading.Thread.start(self)
-
- @property
- def exitcode(self):
- if self._start_called and not self.is_alive():
- return 0
- else:
- return None
-
-#
-#
-#
-
-Process = DummyProcess
-current_process = threading.current_thread
-current_process()._children = weakref.WeakKeyDictionary()
-
-def active_children():
- children = current_process()._children
- for p in list(children):
- if not p.is_alive():
- children.pop(p, None)
- return list(children)
-
-def freeze_support():
- pass
-
-#
-#
-#
-
-class Namespace(object):
+#
+# Support for the API of the multiprocessing package using threads
+#
+# multiprocessing/dummy/__init__.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = [
+ 'Process', 'current_process', 'active_children', 'freeze_support',
+ 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
+ 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue'
+ ]
+
+#
+# Imports
+#
+
+import threading
+import sys
+import weakref
+import array
+
+from .connection import Pipe
+from threading import Lock, RLock, Semaphore, BoundedSemaphore
+from threading import Event, Condition, Barrier
+from queue import Queue
+
+#
+#
+#
+
+class DummyProcess(threading.Thread):
+
+ def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
+ threading.Thread.__init__(self, group, target, name, args, kwargs)
+ self._pid = None
+ self._children = weakref.WeakKeyDictionary()
+ self._start_called = False
+ self._parent = current_process()
+
+ def start(self):
+ if self._parent is not current_process():
+ raise RuntimeError(
+ "Parent is {0!r} but current_process is {1!r}".format(
+ self._parent, current_process()))
+ self._start_called = True
+ if hasattr(self._parent, '_children'):
+ self._parent._children[self] = None
+ threading.Thread.start(self)
+
+ @property
+ def exitcode(self):
+ if self._start_called and not self.is_alive():
+ return 0
+ else:
+ return None
+
+#
+#
+#
+
+Process = DummyProcess
+current_process = threading.current_thread
+current_process()._children = weakref.WeakKeyDictionary()
+
+def active_children():
+ children = current_process()._children
+ for p in list(children):
+ if not p.is_alive():
+ children.pop(p, None)
+ return list(children)
+
+def freeze_support():
+ pass
+
+#
+#
+#
+
+class Namespace(object):
def __init__(self, /, **kwds):
- self.__dict__.update(kwds)
- def __repr__(self):
- items = list(self.__dict__.items())
- temp = []
- for name, value in items:
- if not name.startswith('_'):
- temp.append('%s=%r' % (name, value))
- temp.sort()
- return '%s(%s)' % (self.__class__.__name__, ', '.join(temp))
-
-dict = dict
-list = list
-
-def Array(typecode, sequence, lock=True):
- return array.array(typecode, sequence)
-
-class Value(object):
- def __init__(self, typecode, value, lock=True):
- self._typecode = typecode
- self._value = value
-
- @property
- def value(self):
- return self._value
-
- @value.setter
- def value(self, value):
- self._value = value
-
- def __repr__(self):
- return '<%s(%r, %r)>'%(type(self).__name__,self._typecode,self._value)
-
-def Manager():
- return sys.modules[__name__]
-
-def shutdown():
- pass
-
-def Pool(processes=None, initializer=None, initargs=()):
- from ..pool import ThreadPool
- return ThreadPool(processes, initializer, initargs)
-
-JoinableQueue = Queue
+ self.__dict__.update(kwds)
+ def __repr__(self):
+ items = list(self.__dict__.items())
+ temp = []
+ for name, value in items:
+ if not name.startswith('_'):
+ temp.append('%s=%r' % (name, value))
+ temp.sort()
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(temp))
+
+dict = dict
+list = list
+
+def Array(typecode, sequence, lock=True):
+ return array.array(typecode, sequence)
+
+class Value(object):
+ def __init__(self, typecode, value, lock=True):
+ self._typecode = typecode
+ self._value = value
+
+ @property
+ def value(self):
+ return self._value
+
+ @value.setter
+ def value(self, value):
+ self._value = value
+
+ def __repr__(self):
+ return '<%s(%r, %r)>'%(type(self).__name__,self._typecode,self._value)
+
+def Manager():
+ return sys.modules[__name__]
+
+def shutdown():
+ pass
+
+def Pool(processes=None, initializer=None, initargs=()):
+ from ..pool import ThreadPool
+ return ThreadPool(processes, initializer, initargs)
+
+JoinableQueue = Queue
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/dummy/connection.py b/contrib/tools/python3/src/Lib/multiprocessing/dummy/connection.py
index f0ce320fcf5..b013d021f71 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/dummy/connection.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/dummy/connection.py
@@ -1,75 +1,75 @@
-#
-# Analogue of `multiprocessing.connection` which uses queues instead of sockets
-#
-# multiprocessing/dummy/connection.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-__all__ = [ 'Client', 'Listener', 'Pipe' ]
-
-from queue import Queue
-
-
-families = [None]
-
-
-class Listener(object):
-
- def __init__(self, address=None, family=None, backlog=1):
- self._backlog_queue = Queue(backlog)
-
- def accept(self):
- return Connection(*self._backlog_queue.get())
-
- def close(self):
- self._backlog_queue = None
-
- @property
- def address(self):
- return self._backlog_queue
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- self.close()
-
-
-def Client(address):
- _in, _out = Queue(), Queue()
- address.put((_out, _in))
- return Connection(_in, _out)
-
-
-def Pipe(duplex=True):
- a, b = Queue(), Queue()
- return Connection(a, b), Connection(b, a)
-
-
-class Connection(object):
-
- def __init__(self, _in, _out):
- self._out = _out
- self._in = _in
- self.send = self.send_bytes = _out.put
- self.recv = self.recv_bytes = _in.get
-
- def poll(self, timeout=0.0):
- if self._in.qsize() > 0:
- return True
- if timeout <= 0.0:
- return False
- with self._in.not_empty:
- self._in.not_empty.wait(timeout)
- return self._in.qsize() > 0
-
- def close(self):
- pass
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- self.close()
+#
+# Analogue of `multiprocessing.connection` which uses queues instead of sockets
+#
+# multiprocessing/dummy/connection.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = [ 'Client', 'Listener', 'Pipe' ]
+
+from queue import Queue
+
+
+families = [None]
+
+
+class Listener(object):
+
+ def __init__(self, address=None, family=None, backlog=1):
+ self._backlog_queue = Queue(backlog)
+
+ def accept(self):
+ return Connection(*self._backlog_queue.get())
+
+ def close(self):
+ self._backlog_queue = None
+
+ @property
+ def address(self):
+ return self._backlog_queue
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ self.close()
+
+
+def Client(address):
+ _in, _out = Queue(), Queue()
+ address.put((_out, _in))
+ return Connection(_in, _out)
+
+
+def Pipe(duplex=True):
+ a, b = Queue(), Queue()
+ return Connection(a, b), Connection(b, a)
+
+
+class Connection(object):
+
+ def __init__(self, _in, _out):
+ self._out = _out
+ self._in = _in
+ self.send = self.send_bytes = _out.put
+ self.recv = self.recv_bytes = _in.get
+
+ def poll(self, timeout=0.0):
+ if self._in.qsize() > 0:
+ return True
+ if timeout <= 0.0:
+ return False
+ with self._in.not_empty:
+ self._in.not_empty.wait(timeout)
+ return self._in.qsize() > 0
+
+ def close(self):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ self.close()
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/forkserver.py b/contrib/tools/python3/src/Lib/multiprocessing/forkserver.py
index 22a911a7a29..22c1fbe367f 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/forkserver.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/forkserver.py
@@ -1,44 +1,44 @@
-import errno
-import os
-import selectors
-import signal
-import socket
-import struct
-import sys
-import threading
-import warnings
-
-from . import connection
-from . import process
-from .context import reduction
+import errno
+import os
+import selectors
+import signal
+import socket
+import struct
+import sys
+import threading
+import warnings
+
+from . import connection
+from . import process
+from .context import reduction
from . import resource_tracker
-from . import spawn
-from . import util
-
-__all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
- 'set_forkserver_preload']
-
-#
-#
-#
-
-MAXFDS_TO_SEND = 256
-SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t
-
-#
-# Forkserver class
-#
-
-class ForkServer(object):
-
- def __init__(self):
- self._forkserver_address = None
- self._forkserver_alive_fd = None
- self._forkserver_pid = None
- self._inherited_fds = None
- self._lock = threading.Lock()
- self._preload_modules = ['__main__']
-
+from . import spawn
+from . import util
+
+__all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
+ 'set_forkserver_preload']
+
+#
+#
+#
+
+MAXFDS_TO_SEND = 256
+SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t
+
+#
+# Forkserver class
+#
+
+class ForkServer(object):
+
+ def __init__(self):
+ self._forkserver_address = None
+ self._forkserver_alive_fd = None
+ self._forkserver_pid = None
+ self._inherited_fds = None
+ self._lock = threading.Lock()
+ self._preload_modules = ['__main__']
+
def _stop(self):
# Method used by unit tests to stop the server
with self._lock:
@@ -59,290 +59,290 @@ class ForkServer(object):
os.unlink(self._forkserver_address)
self._forkserver_address = None
- def set_forkserver_preload(self, modules_names):
- '''Set list of module names to try to load in forkserver process.'''
- if not all(type(mod) is str for mod in self._preload_modules):
- raise TypeError('module_names must be a list of strings')
- self._preload_modules = modules_names
-
- def get_inherited_fds(self):
- '''Return list of fds inherited from parent process.
-
- This returns None if the current process was not started by fork
- server.
- '''
- return self._inherited_fds
-
- def connect_to_new_process(self, fds):
- '''Request forkserver to create a child process.
-
- Returns a pair of fds (status_r, data_w). The calling process can read
- the child process's pid and (eventually) its returncode from status_r.
- The calling process should write to data_w the pickled preparation and
- process data.
- '''
- self.ensure_running()
- if len(fds) + 4 >= MAXFDS_TO_SEND:
- raise ValueError('too many fds')
- with socket.socket(socket.AF_UNIX) as client:
- client.connect(self._forkserver_address)
- parent_r, child_w = os.pipe()
- child_r, parent_w = os.pipe()
- allfds = [child_r, child_w, self._forkserver_alive_fd,
+ def set_forkserver_preload(self, modules_names):
+ '''Set list of module names to try to load in forkserver process.'''
+ if not all(type(mod) is str for mod in self._preload_modules):
+ raise TypeError('module_names must be a list of strings')
+ self._preload_modules = modules_names
+
+ def get_inherited_fds(self):
+ '''Return list of fds inherited from parent process.
+
+ This returns None if the current process was not started by fork
+ server.
+ '''
+ return self._inherited_fds
+
+ def connect_to_new_process(self, fds):
+ '''Request forkserver to create a child process.
+
+ Returns a pair of fds (status_r, data_w). The calling process can read
+ the child process's pid and (eventually) its returncode from status_r.
+ The calling process should write to data_w the pickled preparation and
+ process data.
+ '''
+ self.ensure_running()
+ if len(fds) + 4 >= MAXFDS_TO_SEND:
+ raise ValueError('too many fds')
+ with socket.socket(socket.AF_UNIX) as client:
+ client.connect(self._forkserver_address)
+ parent_r, child_w = os.pipe()
+ child_r, parent_w = os.pipe()
+ allfds = [child_r, child_w, self._forkserver_alive_fd,
resource_tracker.getfd()]
- allfds += fds
- try:
- reduction.sendfds(client, allfds)
- return parent_r, parent_w
- except:
- os.close(parent_r)
- os.close(parent_w)
- raise
- finally:
- os.close(child_r)
- os.close(child_w)
-
- def ensure_running(self):
- '''Make sure that a fork server is running.
-
- This can be called from any process. Note that usually a child
- process will just reuse the forkserver started by its parent, so
- ensure_running() will do nothing.
- '''
- with self._lock:
+ allfds += fds
+ try:
+ reduction.sendfds(client, allfds)
+ return parent_r, parent_w
+ except:
+ os.close(parent_r)
+ os.close(parent_w)
+ raise
+ finally:
+ os.close(child_r)
+ os.close(child_w)
+
+ def ensure_running(self):
+ '''Make sure that a fork server is running.
+
+ This can be called from any process. Note that usually a child
+ process will just reuse the forkserver started by its parent, so
+ ensure_running() will do nothing.
+ '''
+ with self._lock:
resource_tracker.ensure_running()
- if self._forkserver_pid is not None:
- # forkserver was launched before, is it still running?
- pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
- if not pid:
- # still alive
- return
- # dead, launch it again
- os.close(self._forkserver_alive_fd)
- self._forkserver_address = None
- self._forkserver_alive_fd = None
- self._forkserver_pid = None
-
- cmd = ('from multiprocessing.forkserver import main; ' +
- 'main(%d, %d, %r, **%r)')
-
- if self._preload_modules:
- desired_keys = {'main_path', 'sys_path'}
- data = spawn.get_preparation_data('ignore')
- data = {x: y for x, y in data.items() if x in desired_keys}
- else:
- data = {}
-
- with socket.socket(socket.AF_UNIX) as listener:
- address = connection.arbitrary_address('AF_UNIX')
- listener.bind(address)
+ if self._forkserver_pid is not None:
+ # forkserver was launched before, is it still running?
+ pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
+ if not pid:
+ # still alive
+ return
+ # dead, launch it again
+ os.close(self._forkserver_alive_fd)
+ self._forkserver_address = None
+ self._forkserver_alive_fd = None
+ self._forkserver_pid = None
+
+ cmd = ('from multiprocessing.forkserver import main; ' +
+ 'main(%d, %d, %r, **%r)')
+
+ if self._preload_modules:
+ desired_keys = {'main_path', 'sys_path'}
+ data = spawn.get_preparation_data('ignore')
+ data = {x: y for x, y in data.items() if x in desired_keys}
+ else:
+ data = {}
+
+ with socket.socket(socket.AF_UNIX) as listener:
+ address = connection.arbitrary_address('AF_UNIX')
+ listener.bind(address)
if not util.is_abstract_socket_namespace(address):
os.chmod(address, 0o600)
- listener.listen()
-
- # all client processes own the write end of the "alive" pipe;
- # when they all terminate the read end becomes ready.
- alive_r, alive_w = os.pipe()
- try:
- fds_to_pass = [listener.fileno(), alive_r]
- cmd %= (listener.fileno(), alive_r, self._preload_modules,
- data)
- exe = spawn.get_executable()
- args = [exe] + util._args_from_interpreter_flags()
- args += ['-c', cmd]
- pid = util.spawnv_passfds(exe, args, fds_to_pass)
- except:
- os.close(alive_w)
- raise
- finally:
- os.close(alive_r)
- self._forkserver_address = address
- self._forkserver_alive_fd = alive_w
- self._forkserver_pid = pid
-
-#
-#
-#
-
-def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
- '''Run forkserver.'''
- if preload:
- if '__main__' in preload and main_path is not None:
- process.current_process()._inheriting = True
- try:
- spawn.import_main_path(main_path)
- finally:
- del process.current_process()._inheriting
- for modname in preload:
- try:
- __import__(modname)
- except ImportError:
- pass
-
- util._close_stdin()
-
- sig_r, sig_w = os.pipe()
- os.set_blocking(sig_r, False)
- os.set_blocking(sig_w, False)
-
- def sigchld_handler(*_unused):
- # Dummy signal handler, doesn't do anything
- pass
-
- handlers = {
- # unblocking SIGCHLD allows the wakeup fd to notify our event loop
- signal.SIGCHLD: sigchld_handler,
- # protect the process from ^C
- signal.SIGINT: signal.SIG_IGN,
- }
- old_handlers = {sig: signal.signal(sig, val)
- for (sig, val) in handlers.items()}
-
- # calling os.write() in the Python signal handler is racy
- signal.set_wakeup_fd(sig_w)
-
- # map child pids to client fds
- pid_to_fd = {}
-
- with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
- selectors.DefaultSelector() as selector:
- _forkserver._forkserver_address = listener.getsockname()
-
- selector.register(listener, selectors.EVENT_READ)
- selector.register(alive_r, selectors.EVENT_READ)
- selector.register(sig_r, selectors.EVENT_READ)
-
- while True:
- try:
- while True:
- rfds = [key.fileobj for (key, events) in selector.select()]
- if rfds:
- break
-
- if alive_r in rfds:
- # EOF because no more client processes left
- assert os.read(alive_r, 1) == b'', "Not at EOF?"
- raise SystemExit
-
- if sig_r in rfds:
- # Got SIGCHLD
- os.read(sig_r, 65536) # exhaust
- while True:
- # Scan for child processes
- try:
- pid, sts = os.waitpid(-1, os.WNOHANG)
- except ChildProcessError:
- break
- if pid == 0:
- break
- child_w = pid_to_fd.pop(pid, None)
- if child_w is not None:
+ listener.listen()
+
+ # all client processes own the write end of the "alive" pipe;
+ # when they all terminate the read end becomes ready.
+ alive_r, alive_w = os.pipe()
+ try:
+ fds_to_pass = [listener.fileno(), alive_r]
+ cmd %= (listener.fileno(), alive_r, self._preload_modules,
+ data)
+ exe = spawn.get_executable()
+ args = [exe] + util._args_from_interpreter_flags()
+ args += ['-c', cmd]
+ pid = util.spawnv_passfds(exe, args, fds_to_pass)
+ except:
+ os.close(alive_w)
+ raise
+ finally:
+ os.close(alive_r)
+ self._forkserver_address = address
+ self._forkserver_alive_fd = alive_w
+ self._forkserver_pid = pid
+
+#
+#
+#
+
+def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
+ '''Run forkserver.'''
+ if preload:
+ if '__main__' in preload and main_path is not None:
+ process.current_process()._inheriting = True
+ try:
+ spawn.import_main_path(main_path)
+ finally:
+ del process.current_process()._inheriting
+ for modname in preload:
+ try:
+ __import__(modname)
+ except ImportError:
+ pass
+
+ util._close_stdin()
+
+ sig_r, sig_w = os.pipe()
+ os.set_blocking(sig_r, False)
+ os.set_blocking(sig_w, False)
+
+ def sigchld_handler(*_unused):
+ # Dummy signal handler, doesn't do anything
+ pass
+
+ handlers = {
+ # unblocking SIGCHLD allows the wakeup fd to notify our event loop
+ signal.SIGCHLD: sigchld_handler,
+ # protect the process from ^C
+ signal.SIGINT: signal.SIG_IGN,
+ }
+ old_handlers = {sig: signal.signal(sig, val)
+ for (sig, val) in handlers.items()}
+
+ # calling os.write() in the Python signal handler is racy
+ signal.set_wakeup_fd(sig_w)
+
+ # map child pids to client fds
+ pid_to_fd = {}
+
+ with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
+ selectors.DefaultSelector() as selector:
+ _forkserver._forkserver_address = listener.getsockname()
+
+ selector.register(listener, selectors.EVENT_READ)
+ selector.register(alive_r, selectors.EVENT_READ)
+ selector.register(sig_r, selectors.EVENT_READ)
+
+ while True:
+ try:
+ while True:
+ rfds = [key.fileobj for (key, events) in selector.select()]
+ if rfds:
+ break
+
+ if alive_r in rfds:
+ # EOF because no more client processes left
+ assert os.read(alive_r, 1) == b'', "Not at EOF?"
+ raise SystemExit
+
+ if sig_r in rfds:
+ # Got SIGCHLD
+ os.read(sig_r, 65536) # exhaust
+ while True:
+ # Scan for child processes
+ try:
+ pid, sts = os.waitpid(-1, os.WNOHANG)
+ except ChildProcessError:
+ break
+ if pid == 0:
+ break
+ child_w = pid_to_fd.pop(pid, None)
+ if child_w is not None:
returncode = os.waitstatus_to_exitcode(sts)
- # Send exit code to client process
- try:
- write_signed(child_w, returncode)
- except BrokenPipeError:
- # client vanished
- pass
- os.close(child_w)
- else:
- # This shouldn't happen really
- warnings.warn('forkserver: waitpid returned '
- 'unexpected pid %d' % pid)
-
- if listener in rfds:
- # Incoming fork request
- with listener.accept()[0] as s:
- # Receive fds from client
- fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
- if len(fds) > MAXFDS_TO_SEND:
- raise RuntimeError(
- "Too many ({0:n}) fds to send".format(
- len(fds)))
- child_r, child_w, *fds = fds
- s.close()
- pid = os.fork()
- if pid == 0:
- # Child
- code = 1
- try:
- listener.close()
- selector.close()
- unused_fds = [alive_r, child_w, sig_r, sig_w]
- unused_fds.extend(pid_to_fd.values())
- code = _serve_one(child_r, fds,
- unused_fds,
- old_handlers)
- except Exception:
- sys.excepthook(*sys.exc_info())
- sys.stderr.flush()
- finally:
- os._exit(code)
- else:
- # Send pid to client process
- try:
- write_signed(child_w, pid)
- except BrokenPipeError:
- # client vanished
- pass
- pid_to_fd[pid] = child_w
- os.close(child_r)
- for fd in fds:
- os.close(fd)
-
- except OSError as e:
- if e.errno != errno.ECONNABORTED:
- raise
-
-
-def _serve_one(child_r, fds, unused_fds, handlers):
- # close unnecessary stuff and reset signal handlers
- signal.set_wakeup_fd(-1)
- for sig, val in handlers.items():
- signal.signal(sig, val)
- for fd in unused_fds:
- os.close(fd)
-
- (_forkserver._forkserver_alive_fd,
+ # Send exit code to client process
+ try:
+ write_signed(child_w, returncode)
+ except BrokenPipeError:
+ # client vanished
+ pass
+ os.close(child_w)
+ else:
+ # This shouldn't happen really
+ warnings.warn('forkserver: waitpid returned '
+ 'unexpected pid %d' % pid)
+
+ if listener in rfds:
+ # Incoming fork request
+ with listener.accept()[0] as s:
+ # Receive fds from client
+ fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
+ if len(fds) > MAXFDS_TO_SEND:
+ raise RuntimeError(
+ "Too many ({0:n}) fds to send".format(
+ len(fds)))
+ child_r, child_w, *fds = fds
+ s.close()
+ pid = os.fork()
+ if pid == 0:
+ # Child
+ code = 1
+ try:
+ listener.close()
+ selector.close()
+ unused_fds = [alive_r, child_w, sig_r, sig_w]
+ unused_fds.extend(pid_to_fd.values())
+ code = _serve_one(child_r, fds,
+ unused_fds,
+ old_handlers)
+ except Exception:
+ sys.excepthook(*sys.exc_info())
+ sys.stderr.flush()
+ finally:
+ os._exit(code)
+ else:
+ # Send pid to client process
+ try:
+ write_signed(child_w, pid)
+ except BrokenPipeError:
+ # client vanished
+ pass
+ pid_to_fd[pid] = child_w
+ os.close(child_r)
+ for fd in fds:
+ os.close(fd)
+
+ except OSError as e:
+ if e.errno != errno.ECONNABORTED:
+ raise
+
+
+def _serve_one(child_r, fds, unused_fds, handlers):
+ # close unnecessary stuff and reset signal handlers
+ signal.set_wakeup_fd(-1)
+ for sig, val in handlers.items():
+ signal.signal(sig, val)
+ for fd in unused_fds:
+ os.close(fd)
+
+ (_forkserver._forkserver_alive_fd,
resource_tracker._resource_tracker._fd,
- *_forkserver._inherited_fds) = fds
-
- # Run process object received over pipe
+ *_forkserver._inherited_fds) = fds
+
+ # Run process object received over pipe
parent_sentinel = os.dup(child_r)
code = spawn._main(child_r, parent_sentinel)
-
- return code
-
-
-#
-# Read and write signed numbers
-#
-
-def read_signed(fd):
- data = b''
- length = SIGNED_STRUCT.size
- while len(data) < length:
- s = os.read(fd, length - len(data))
- if not s:
- raise EOFError('unexpected EOF')
- data += s
- return SIGNED_STRUCT.unpack(data)[0]
-
-def write_signed(fd, n):
- msg = SIGNED_STRUCT.pack(n)
- while msg:
- nbytes = os.write(fd, msg)
- if nbytes == 0:
- raise RuntimeError('should not get here')
- msg = msg[nbytes:]
-
-#
-#
-#
-
-_forkserver = ForkServer()
-ensure_running = _forkserver.ensure_running
-get_inherited_fds = _forkserver.get_inherited_fds
-connect_to_new_process = _forkserver.connect_to_new_process
-set_forkserver_preload = _forkserver.set_forkserver_preload
+
+ return code
+
+
+#
+# Read and write signed numbers
+#
+
+def read_signed(fd):
+ data = b''
+ length = SIGNED_STRUCT.size
+ while len(data) < length:
+ s = os.read(fd, length - len(data))
+ if not s:
+ raise EOFError('unexpected EOF')
+ data += s
+ return SIGNED_STRUCT.unpack(data)[0]
+
+def write_signed(fd, n):
+ msg = SIGNED_STRUCT.pack(n)
+ while msg:
+ nbytes = os.write(fd, msg)
+ if nbytes == 0:
+ raise RuntimeError('should not get here')
+ msg = msg[nbytes:]
+
+#
+#
+#
+
+_forkserver = ForkServer()
+ensure_running = _forkserver.ensure_running
+get_inherited_fds = _forkserver.get_inherited_fds
+connect_to_new_process = _forkserver.connect_to_new_process
+set_forkserver_preload = _forkserver.set_forkserver_preload
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/heap.py b/contrib/tools/python3/src/Lib/multiprocessing/heap.py
index 6217dfe1268..982a1ffc376 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/heap.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/heap.py
@@ -1,160 +1,160 @@
-#
-# Module which supports allocation of memory from an mmap
-#
-# multiprocessing/heap.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-import bisect
+#
+# Module which supports allocation of memory from an mmap
+#
+# multiprocessing/heap.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import bisect
from collections import defaultdict
-import mmap
-import os
-import sys
-import tempfile
-import threading
-
-from .context import reduction, assert_spawning
-from . import util
-
-__all__ = ['BufferWrapper']
-
-#
-# Inheritable class which wraps an mmap, and from which blocks can be allocated
-#
-
-if sys.platform == 'win32':
-
- import _winapi
-
- class Arena(object):
+import mmap
+import os
+import sys
+import tempfile
+import threading
+
+from .context import reduction, assert_spawning
+from . import util
+
+__all__ = ['BufferWrapper']
+
+#
+# Inheritable class which wraps an mmap, and from which blocks can be allocated
+#
+
+if sys.platform == 'win32':
+
+ import _winapi
+
+ class Arena(object):
"""
A shared memory area backed by anonymous memory (Windows).
"""
-
- _rand = tempfile._RandomNameSequence()
-
- def __init__(self, size):
- self.size = size
- for i in range(100):
- name = 'pym-%d-%s' % (os.getpid(), next(self._rand))
- buf = mmap.mmap(-1, size, tagname=name)
- if _winapi.GetLastError() == 0:
- break
- # We have reopened a preexisting mmap.
- buf.close()
- else:
- raise FileExistsError('Cannot find name for new mmap')
- self.name = name
- self.buffer = buf
- self._state = (self.size, self.name)
-
- def __getstate__(self):
- assert_spawning(self)
- return self._state
-
- def __setstate__(self, state):
- self.size, self.name = self._state = state
+
+ _rand = tempfile._RandomNameSequence()
+
+ def __init__(self, size):
+ self.size = size
+ for i in range(100):
+ name = 'pym-%d-%s' % (os.getpid(), next(self._rand))
+ buf = mmap.mmap(-1, size, tagname=name)
+ if _winapi.GetLastError() == 0:
+ break
+ # We have reopened a preexisting mmap.
+ buf.close()
+ else:
+ raise FileExistsError('Cannot find name for new mmap')
+ self.name = name
+ self.buffer = buf
+ self._state = (self.size, self.name)
+
+ def __getstate__(self):
+ assert_spawning(self)
+ return self._state
+
+ def __setstate__(self, state):
+ self.size, self.name = self._state = state
# Reopen existing mmap
- self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
- # XXX Temporarily preventing buildbot failures while determining
- # XXX the correct long-term fix. See issue 23060
- #assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS
-
-else:
-
- class Arena(object):
+ self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
+ # XXX Temporarily preventing buildbot failures while determining
+ # XXX the correct long-term fix. See issue 23060
+ #assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS
+
+else:
+
+ class Arena(object):
"""
A shared memory area backed by a temporary file (POSIX).
"""
- if sys.platform == 'linux':
- _dir_candidates = ['/dev/shm']
- else:
- _dir_candidates = []
-
- def __init__(self, size, fd=-1):
- self.size = size
- self.fd = fd
- if fd == -1:
+ if sys.platform == 'linux':
+ _dir_candidates = ['/dev/shm']
+ else:
+ _dir_candidates = []
+
+ def __init__(self, size, fd=-1):
+ self.size = size
+ self.fd = fd
+ if fd == -1:
# Arena is created anew (if fd != -1, it means we're coming
# from rebuild_arena() below)
- self.fd, name = tempfile.mkstemp(
- prefix='pym-%d-'%os.getpid(),
- dir=self._choose_dir(size))
- os.unlink(name)
- util.Finalize(self, os.close, (self.fd,))
- os.ftruncate(self.fd, size)
- self.buffer = mmap.mmap(self.fd, self.size)
-
- def _choose_dir(self, size):
- # Choose a non-storage backed directory if possible,
- # to improve performance
- for d in self._dir_candidates:
- st = os.statvfs(d)
- if st.f_bavail * st.f_frsize >= size: # enough free space?
- return d
- return util.get_temp_dir()
-
- def reduce_arena(a):
- if a.fd == -1:
- raise ValueError('Arena is unpicklable because '
- 'forking was enabled when it was created')
- return rebuild_arena, (a.size, reduction.DupFd(a.fd))
-
- def rebuild_arena(size, dupfd):
- return Arena(size, dupfd.detach())
-
- reduction.register(Arena, reduce_arena)
-
-#
-# Class allowing allocation of chunks of memory from arenas
-#
-
-class Heap(object):
-
+ self.fd, name = tempfile.mkstemp(
+ prefix='pym-%d-'%os.getpid(),
+ dir=self._choose_dir(size))
+ os.unlink(name)
+ util.Finalize(self, os.close, (self.fd,))
+ os.ftruncate(self.fd, size)
+ self.buffer = mmap.mmap(self.fd, self.size)
+
+ def _choose_dir(self, size):
+ # Choose a non-storage backed directory if possible,
+ # to improve performance
+ for d in self._dir_candidates:
+ st = os.statvfs(d)
+ if st.f_bavail * st.f_frsize >= size: # enough free space?
+ return d
+ return util.get_temp_dir()
+
+ def reduce_arena(a):
+ if a.fd == -1:
+ raise ValueError('Arena is unpicklable because '
+ 'forking was enabled when it was created')
+ return rebuild_arena, (a.size, reduction.DupFd(a.fd))
+
+ def rebuild_arena(size, dupfd):
+ return Arena(size, dupfd.detach())
+
+ reduction.register(Arena, reduce_arena)
+
+#
+# Class allowing allocation of chunks of memory from arenas
+#
+
+class Heap(object):
+
# Minimum malloc() alignment
- _alignment = 8
-
+ _alignment = 8
+
_DISCARD_FREE_SPACE_LARGER_THAN = 4 * 1024 ** 2 # 4 MB
_DOUBLE_ARENA_SIZE_UNTIL = 4 * 1024 ** 2
- def __init__(self, size=mmap.PAGESIZE):
- self._lastpid = os.getpid()
- self._lock = threading.Lock()
+ def __init__(self, size=mmap.PAGESIZE):
+ self._lastpid = os.getpid()
+ self._lock = threading.Lock()
# Current arena allocation size
- self._size = size
+ self._size = size
# A sorted list of available block sizes in arenas
- self._lengths = []
+ self._lengths = []
# Free block management:
# - map each block size to a list of `(Arena, start, stop)` blocks
- self._len_to_seq = {}
+ self._len_to_seq = {}
# - map `(Arena, start)` tuple to the `(Arena, start, stop)` block
# starting at that offset
- self._start_to_block = {}
+ self._start_to_block = {}
# - map `(Arena, stop)` tuple to the `(Arena, start, stop)` block
# ending at that offset
- self._stop_to_block = {}
+ self._stop_to_block = {}
# Map arenas to their `(Arena, start, stop)` blocks in use
self._allocated_blocks = defaultdict(set)
- self._arenas = []
+ self._arenas = []
# List of pending blocks to free - see comment in free() below
- self._pending_free_blocks = []
-
+ self._pending_free_blocks = []
+
# Statistics
self._n_mallocs = 0
self._n_frees = 0
- @staticmethod
- def _roundup(n, alignment):
- # alignment must be a power of 2
- mask = alignment - 1
- return (n + mask) & ~mask
-
+ @staticmethod
+ def _roundup(n, alignment):
+ # alignment must be a power of 2
+ mask = alignment - 1
+ return (n + mask) & ~mask
+
def _new_arena(self, size):
# Create a new arena with at least the given *size*
length = self._roundup(max(self._size, size), mmap.PAGESIZE)
@@ -185,68 +185,68 @@ class Heap(object):
del self._len_to_seq[length]
self._lengths.remove(length)
- def _malloc(self, size):
- # returns a large enough block -- it might be much larger
- i = bisect.bisect_left(self._lengths, size)
- if i == len(self._lengths):
+ def _malloc(self, size):
+ # returns a large enough block -- it might be much larger
+ i = bisect.bisect_left(self._lengths, size)
+ if i == len(self._lengths):
return self._new_arena(size)
- else:
- length = self._lengths[i]
- seq = self._len_to_seq[length]
- block = seq.pop()
- if not seq:
- del self._len_to_seq[length], self._lengths[i]
-
- (arena, start, stop) = block
- del self._start_to_block[(arena, start)]
- del self._stop_to_block[(arena, stop)]
- return block
-
+ else:
+ length = self._lengths[i]
+ seq = self._len_to_seq[length]
+ block = seq.pop()
+ if not seq:
+ del self._len_to_seq[length], self._lengths[i]
+
+ (arena, start, stop) = block
+ del self._start_to_block[(arena, start)]
+ del self._stop_to_block[(arena, stop)]
+ return block
+
def _add_free_block(self, block):
# make block available and try to merge with its neighbours in the arena
- (arena, start, stop) = block
-
- try:
- prev_block = self._stop_to_block[(arena, start)]
- except KeyError:
- pass
- else:
- start, _ = self._absorb(prev_block)
-
- try:
- next_block = self._start_to_block[(arena, stop)]
- except KeyError:
- pass
- else:
- _, stop = self._absorb(next_block)
-
- block = (arena, start, stop)
- length = stop - start
-
- try:
- self._len_to_seq[length].append(block)
- except KeyError:
- self._len_to_seq[length] = [block]
- bisect.insort(self._lengths, length)
-
- self._start_to_block[(arena, start)] = block
- self._stop_to_block[(arena, stop)] = block
-
- def _absorb(self, block):
- # deregister this block so it can be merged with a neighbour
- (arena, start, stop) = block
- del self._start_to_block[(arena, start)]
- del self._stop_to_block[(arena, stop)]
-
- length = stop - start
- seq = self._len_to_seq[length]
- seq.remove(block)
- if not seq:
- del self._len_to_seq[length]
- self._lengths.remove(length)
-
- return start, stop
-
+ (arena, start, stop) = block
+
+ try:
+ prev_block = self._stop_to_block[(arena, start)]
+ except KeyError:
+ pass
+ else:
+ start, _ = self._absorb(prev_block)
+
+ try:
+ next_block = self._start_to_block[(arena, stop)]
+ except KeyError:
+ pass
+ else:
+ _, stop = self._absorb(next_block)
+
+ block = (arena, start, stop)
+ length = stop - start
+
+ try:
+ self._len_to_seq[length].append(block)
+ except KeyError:
+ self._len_to_seq[length] = [block]
+ bisect.insort(self._lengths, length)
+
+ self._start_to_block[(arena, start)] = block
+ self._stop_to_block[(arena, stop)] = block
+
+ def _absorb(self, block):
+ # deregister this block so it can be merged with a neighbour
+ (arena, start, stop) = block
+ del self._start_to_block[(arena, start)]
+ del self._stop_to_block[(arena, stop)]
+
+ length = stop - start
+ seq = self._len_to_seq[length]
+ seq.remove(block)
+ if not seq:
+ del self._len_to_seq[length]
+ self._lengths.remove(length)
+
+ return start, stop
+
def _remove_allocated_block(self, block):
arena, start, stop = block
blocks = self._allocated_blocks[arena]
@@ -255,58 +255,58 @@ class Heap(object):
# Arena is entirely free, discard it from this process
self._discard_arena(arena)
- def _free_pending_blocks(self):
- # Free all the blocks in the pending list - called with the lock held.
- while True:
- try:
- block = self._pending_free_blocks.pop()
- except IndexError:
- break
+ def _free_pending_blocks(self):
+ # Free all the blocks in the pending list - called with the lock held.
+ while True:
+ try:
+ block = self._pending_free_blocks.pop()
+ except IndexError:
+ break
self._add_free_block(block)
self._remove_allocated_block(block)
-
- def free(self, block):
- # free a block returned by malloc()
- # Since free() can be called asynchronously by the GC, it could happen
- # that it's called while self._lock is held: in that case,
- # self._lock.acquire() would deadlock (issue #12352). To avoid that, a
- # trylock is used instead, and if the lock can't be acquired
- # immediately, the block is added to a list of blocks to be freed
- # synchronously sometimes later from malloc() or free(), by calling
- # _free_pending_blocks() (appending and retrieving from a list is not
+
+ def free(self, block):
+ # free a block returned by malloc()
+ # Since free() can be called asynchronously by the GC, it could happen
+ # that it's called while self._lock is held: in that case,
+ # self._lock.acquire() would deadlock (issue #12352). To avoid that, a
+ # trylock is used instead, and if the lock can't be acquired
+ # immediately, the block is added to a list of blocks to be freed
+ # synchronously sometimes later from malloc() or free(), by calling
+ # _free_pending_blocks() (appending and retrieving from a list is not
# strictly thread-safe but under CPython it's atomic thanks to the GIL).
- if os.getpid() != self._lastpid:
- raise ValueError(
- "My pid ({0:n}) is not last pid {1:n}".format(
- os.getpid(),self._lastpid))
- if not self._lock.acquire(False):
- # can't acquire the lock right now, add the block to the list of
- # pending blocks to free
- self._pending_free_blocks.append(block)
- else:
- # we hold the lock
- try:
+ if os.getpid() != self._lastpid:
+ raise ValueError(
+ "My pid ({0:n}) is not last pid {1:n}".format(
+ os.getpid(),self._lastpid))
+ if not self._lock.acquire(False):
+ # can't acquire the lock right now, add the block to the list of
+ # pending blocks to free
+ self._pending_free_blocks.append(block)
+ else:
+ # we hold the lock
+ try:
self._n_frees += 1
- self._free_pending_blocks()
+ self._free_pending_blocks()
self._add_free_block(block)
self._remove_allocated_block(block)
- finally:
- self._lock.release()
-
- def malloc(self, size):
- # return a block of right size (possibly rounded up)
- if size < 0:
- raise ValueError("Size {0:n} out of range".format(size))
- if sys.maxsize <= size:
- raise OverflowError("Size {0:n} too large".format(size))
- if os.getpid() != self._lastpid:
- self.__init__() # reinitialize after fork
- with self._lock:
+ finally:
+ self._lock.release()
+
+ def malloc(self, size):
+ # return a block of right size (possibly rounded up)
+ if size < 0:
+ raise ValueError("Size {0:n} out of range".format(size))
+ if sys.maxsize <= size:
+ raise OverflowError("Size {0:n} too large".format(size))
+ if os.getpid() != self._lastpid:
+ self.__init__() # reinitialize after fork
+ with self._lock:
self._n_mallocs += 1
# allow pending blocks to be marked available
- self._free_pending_blocks()
+ self._free_pending_blocks()
size = self._roundup(max(size, 1), self._alignment)
- (arena, start, stop) = self._malloc(size)
+ (arena, start, stop) = self._malloc(size)
real_stop = start + size
if real_stop < stop:
# if the returned block is larger than necessary, mark
@@ -314,24 +314,24 @@ class Heap(object):
self._add_free_block((arena, real_stop, stop))
self._allocated_blocks[arena].add((start, real_stop))
return (arena, start, real_stop)
-
-#
+
+#
# Class wrapping a block allocated out of a Heap -- can be inherited by child process
-#
-
-class BufferWrapper(object):
-
- _heap = Heap()
-
- def __init__(self, size):
- if size < 0:
- raise ValueError("Size {0:n} out of range".format(size))
- if sys.maxsize <= size:
- raise OverflowError("Size {0:n} too large".format(size))
- block = BufferWrapper._heap.malloc(size)
- self._state = (block, size)
- util.Finalize(self, BufferWrapper._heap.free, args=(block,))
-
- def create_memoryview(self):
- (arena, start, stop), size = self._state
- return memoryview(arena.buffer)[start:start+size]
+#
+
+class BufferWrapper(object):
+
+ _heap = Heap()
+
+ def __init__(self, size):
+ if size < 0:
+ raise ValueError("Size {0:n} out of range".format(size))
+ if sys.maxsize <= size:
+ raise OverflowError("Size {0:n} too large".format(size))
+ block = BufferWrapper._heap.malloc(size)
+ self._state = (block, size)
+ util.Finalize(self, BufferWrapper._heap.free, args=(block,))
+
+ def create_memoryview(self):
+ (arena, start, stop), size = self._state
+ return memoryview(arena.buffer)[start:start+size]
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/managers.py b/contrib/tools/python3/src/Lib/multiprocessing/managers.py
index dfa566c6fc3..263dcf53fc4 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/managers.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/managers.py
@@ -1,37 +1,37 @@
-#
+#
# Module providing manager classes for dealing
-# with shared objects
-#
-# multiprocessing/managers.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
+# with shared objects
+#
+# multiprocessing/managers.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
-
-#
-# Imports
-#
-
-import sys
-import threading
+
+#
+# Imports
+#
+
+import sys
+import threading
import signal
-import array
-import queue
-import time
+import array
+import queue
+import time
import types
import os
from os import getpid
-
-from traceback import format_exc
-
-from . import connection
-from .context import reduction, get_spawning_popen, ProcessError
-from . import pool
-from . import process
-from . import util
-from . import get_context
+
+from traceback import format_exc
+
+from . import connection
+from .context import reduction, get_spawning_popen, ProcessError
+from . import pool
+from . import process
+from . import util
+from . import get_context
try:
from . import shared_memory
except ImportError:
@@ -39,1183 +39,1183 @@ except ImportError:
else:
HAS_SHMEM = True
__all__.append('SharedMemoryManager')
-
-#
-# Register some things for pickling
-#
-
-def reduce_array(a):
- return array.array, (a.typecode, a.tobytes())
-reduction.register(array.array, reduce_array)
-
-view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
-if view_types[0] is not list: # only needed in Py3.0
- def rebuild_as_list(obj):
- return list, (list(obj),)
- for view_type in view_types:
- reduction.register(view_type, rebuild_as_list)
-
-#
-# Type for identifying shared objects
-#
-
-class Token(object):
- '''
+
+#
+# Register some things for pickling
+#
+
+def reduce_array(a):
+ return array.array, (a.typecode, a.tobytes())
+reduction.register(array.array, reduce_array)
+
+view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
+if view_types[0] is not list: # only needed in Py3.0
+ def rebuild_as_list(obj):
+ return list, (list(obj),)
+ for view_type in view_types:
+ reduction.register(view_type, rebuild_as_list)
+
+#
+# Type for identifying shared objects
+#
+
+class Token(object):
+ '''
Type to uniquely identify a shared object
- '''
- __slots__ = ('typeid', 'address', 'id')
-
- def __init__(self, typeid, address, id):
- (self.typeid, self.address, self.id) = (typeid, address, id)
-
- def __getstate__(self):
- return (self.typeid, self.address, self.id)
-
- def __setstate__(self, state):
- (self.typeid, self.address, self.id) = state
-
- def __repr__(self):
- return '%s(typeid=%r, address=%r, id=%r)' % \
- (self.__class__.__name__, self.typeid, self.address, self.id)
-
-#
-# Function for communication with a manager's server process
-#
-
-def dispatch(c, id, methodname, args=(), kwds={}):
- '''
- Send a message to manager using connection `c` and return response
- '''
- c.send((id, methodname, args, kwds))
- kind, result = c.recv()
- if kind == '#RETURN':
- return result
- raise convert_to_error(kind, result)
-
-def convert_to_error(kind, result):
- if kind == '#ERROR':
- return result
- elif kind in ('#TRACEBACK', '#UNSERIALIZABLE'):
- if not isinstance(result, str):
- raise TypeError(
- "Result {0!r} (kind '{1}') type is {2}, not str".format(
- result, kind, type(result)))
- if kind == '#UNSERIALIZABLE':
- return RemoteError('Unserializable message: %s\n' % result)
- else:
- return RemoteError(result)
- else:
- return ValueError('Unrecognized message type {!r}'.format(kind))
-
-class RemoteError(Exception):
- def __str__(self):
- return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75)
-
-#
-# Functions for finding the method names of an object
-#
-
-def all_methods(obj):
- '''
- Return a list of names of methods of `obj`
- '''
- temp = []
- for name in dir(obj):
- func = getattr(obj, name)
- if callable(func):
- temp.append(name)
- return temp
-
-def public_methods(obj):
- '''
- Return a list of names of methods of `obj` which do not start with '_'
- '''
- return [name for name in all_methods(obj) if name[0] != '_']
-
-#
-# Server which is run in a process controlled by a manager
-#
-
-class Server(object):
- '''
- Server class which runs in a process controlled by a manager object
- '''
- public = ['shutdown', 'create', 'accept_connection', 'get_methods',
- 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']
-
- def __init__(self, registry, address, authkey, serializer):
- if not isinstance(authkey, bytes):
- raise TypeError(
- "Authkey {0!r} is type {1!s}, not bytes".format(
- authkey, type(authkey)))
- self.registry = registry
- self.authkey = process.AuthenticationString(authkey)
- Listener, Client = listener_client[serializer]
-
- # do authentication later
- self.listener = Listener(address=address, backlog=16)
- self.address = self.listener.address
-
- self.id_to_obj = {'0': (None, ())}
- self.id_to_refcount = {}
- self.id_to_local_proxy_obj = {}
- self.mutex = threading.Lock()
-
- def serve_forever(self):
- '''
- Run the server forever
- '''
- self.stop_event = threading.Event()
- process.current_process()._manager_server = self
- try:
- accepter = threading.Thread(target=self.accepter)
- accepter.daemon = True
- accepter.start()
- try:
- while not self.stop_event.is_set():
- self.stop_event.wait(1)
- except (KeyboardInterrupt, SystemExit):
- pass
- finally:
- if sys.stdout != sys.__stdout__: # what about stderr?
- util.debug('resetting stdout, stderr')
- sys.stdout = sys.__stdout__
- sys.stderr = sys.__stderr__
- sys.exit(0)
-
- def accepter(self):
- while True:
- try:
- c = self.listener.accept()
- except OSError:
- continue
- t = threading.Thread(target=self.handle_request, args=(c,))
- t.daemon = True
- t.start()
-
- def handle_request(self, c):
- '''
- Handle a new connection
- '''
- funcname = result = request = None
- try:
- connection.deliver_challenge(c, self.authkey)
- connection.answer_challenge(c, self.authkey)
- request = c.recv()
- ignore, funcname, args, kwds = request
- assert funcname in self.public, '%r unrecognized' % funcname
- func = getattr(self, funcname)
- except Exception:
- msg = ('#TRACEBACK', format_exc())
- else:
- try:
- result = func(c, *args, **kwds)
- except Exception:
- msg = ('#TRACEBACK', format_exc())
- else:
- msg = ('#RETURN', result)
- try:
- c.send(msg)
- except Exception as e:
- try:
- c.send(('#TRACEBACK', format_exc()))
- except Exception:
- pass
- util.info('Failure to send message: %r', msg)
- util.info(' ... request was %r', request)
- util.info(' ... exception was %r', e)
-
- c.close()
-
- def serve_client(self, conn):
- '''
- Handle requests from the proxies in a particular process/thread
- '''
- util.debug('starting server thread to service %r',
- threading.current_thread().name)
-
- recv = conn.recv
- send = conn.send
- id_to_obj = self.id_to_obj
-
- while not self.stop_event.is_set():
-
- try:
- methodname = obj = None
- request = recv()
- ident, methodname, args, kwds = request
- try:
- obj, exposed, gettypeid = id_to_obj[ident]
- except KeyError as ke:
- try:
- obj, exposed, gettypeid = \
- self.id_to_local_proxy_obj[ident]
+ '''
+ __slots__ = ('typeid', 'address', 'id')
+
+ def __init__(self, typeid, address, id):
+ (self.typeid, self.address, self.id) = (typeid, address, id)
+
+ def __getstate__(self):
+ return (self.typeid, self.address, self.id)
+
+ def __setstate__(self, state):
+ (self.typeid, self.address, self.id) = state
+
+ def __repr__(self):
+ return '%s(typeid=%r, address=%r, id=%r)' % \
+ (self.__class__.__name__, self.typeid, self.address, self.id)
+
+#
+# Function for communication with a manager's server process
+#
+
+def dispatch(c, id, methodname, args=(), kwds={}):
+ '''
+ Send a message to manager using connection `c` and return response
+ '''
+ c.send((id, methodname, args, kwds))
+ kind, result = c.recv()
+ if kind == '#RETURN':
+ return result
+ raise convert_to_error(kind, result)
+
+def convert_to_error(kind, result):
+ if kind == '#ERROR':
+ return result
+ elif kind in ('#TRACEBACK', '#UNSERIALIZABLE'):
+ if not isinstance(result, str):
+ raise TypeError(
+ "Result {0!r} (kind '{1}') type is {2}, not str".format(
+ result, kind, type(result)))
+ if kind == '#UNSERIALIZABLE':
+ return RemoteError('Unserializable message: %s\n' % result)
+ else:
+ return RemoteError(result)
+ else:
+ return ValueError('Unrecognized message type {!r}'.format(kind))
+
+class RemoteError(Exception):
+ def __str__(self):
+ return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75)
+
+#
+# Functions for finding the method names of an object
+#
+
+def all_methods(obj):
+ '''
+ Return a list of names of methods of `obj`
+ '''
+ temp = []
+ for name in dir(obj):
+ func = getattr(obj, name)
+ if callable(func):
+ temp.append(name)
+ return temp
+
+def public_methods(obj):
+ '''
+ Return a list of names of methods of `obj` which do not start with '_'
+ '''
+ return [name for name in all_methods(obj) if name[0] != '_']
+
+#
+# Server which is run in a process controlled by a manager
+#
+
+class Server(object):
+ '''
+ Server class which runs in a process controlled by a manager object
+ '''
+ public = ['shutdown', 'create', 'accept_connection', 'get_methods',
+ 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']
+
+ def __init__(self, registry, address, authkey, serializer):
+ if not isinstance(authkey, bytes):
+ raise TypeError(
+ "Authkey {0!r} is type {1!s}, not bytes".format(
+ authkey, type(authkey)))
+ self.registry = registry
+ self.authkey = process.AuthenticationString(authkey)
+ Listener, Client = listener_client[serializer]
+
+ # do authentication later
+ self.listener = Listener(address=address, backlog=16)
+ self.address = self.listener.address
+
+ self.id_to_obj = {'0': (None, ())}
+ self.id_to_refcount = {}
+ self.id_to_local_proxy_obj = {}
+ self.mutex = threading.Lock()
+
+ def serve_forever(self):
+ '''
+ Run the server forever
+ '''
+ self.stop_event = threading.Event()
+ process.current_process()._manager_server = self
+ try:
+ accepter = threading.Thread(target=self.accepter)
+ accepter.daemon = True
+ accepter.start()
+ try:
+ while not self.stop_event.is_set():
+ self.stop_event.wait(1)
+ except (KeyboardInterrupt, SystemExit):
+ pass
+ finally:
+ if sys.stdout != sys.__stdout__: # what about stderr?
+ util.debug('resetting stdout, stderr')
+ sys.stdout = sys.__stdout__
+ sys.stderr = sys.__stderr__
+ sys.exit(0)
+
+ def accepter(self):
+ while True:
+ try:
+ c = self.listener.accept()
+ except OSError:
+ continue
+ t = threading.Thread(target=self.handle_request, args=(c,))
+ t.daemon = True
+ t.start()
+
+ def handle_request(self, c):
+ '''
+ Handle a new connection
+ '''
+ funcname = result = request = None
+ try:
+ connection.deliver_challenge(c, self.authkey)
+ connection.answer_challenge(c, self.authkey)
+ request = c.recv()
+ ignore, funcname, args, kwds = request
+ assert funcname in self.public, '%r unrecognized' % funcname
+ func = getattr(self, funcname)
+ except Exception:
+ msg = ('#TRACEBACK', format_exc())
+ else:
+ try:
+ result = func(c, *args, **kwds)
+ except Exception:
+ msg = ('#TRACEBACK', format_exc())
+ else:
+ msg = ('#RETURN', result)
+ try:
+ c.send(msg)
+ except Exception as e:
+ try:
+ c.send(('#TRACEBACK', format_exc()))
+ except Exception:
+ pass
+ util.info('Failure to send message: %r', msg)
+ util.info(' ... request was %r', request)
+ util.info(' ... exception was %r', e)
+
+ c.close()
+
+ def serve_client(self, conn):
+ '''
+ Handle requests from the proxies in a particular process/thread
+ '''
+ util.debug('starting server thread to service %r',
+ threading.current_thread().name)
+
+ recv = conn.recv
+ send = conn.send
+ id_to_obj = self.id_to_obj
+
+ while not self.stop_event.is_set():
+
+ try:
+ methodname = obj = None
+ request = recv()
+ ident, methodname, args, kwds = request
+ try:
+ obj, exposed, gettypeid = id_to_obj[ident]
+ except KeyError as ke:
+ try:
+ obj, exposed, gettypeid = \
+ self.id_to_local_proxy_obj[ident]
except KeyError:
- raise ke
-
- if methodname not in exposed:
- raise AttributeError(
- 'method %r of %r object is not in exposed=%r' %
- (methodname, type(obj), exposed)
- )
-
- function = getattr(obj, methodname)
-
- try:
- res = function(*args, **kwds)
- except Exception as e:
- msg = ('#ERROR', e)
- else:
- typeid = gettypeid and gettypeid.get(methodname, None)
- if typeid:
- rident, rexposed = self.create(conn, typeid, res)
- token = Token(typeid, self.address, rident)
- msg = ('#PROXY', (rexposed, token))
- else:
- msg = ('#RETURN', res)
-
- except AttributeError:
- if methodname is None:
- msg = ('#TRACEBACK', format_exc())
- else:
- try:
- fallback_func = self.fallback_mapping[methodname]
- result = fallback_func(
- self, conn, ident, obj, *args, **kwds
- )
- msg = ('#RETURN', result)
- except Exception:
- msg = ('#TRACEBACK', format_exc())
-
- except EOFError:
- util.debug('got EOF -- exiting thread serving %r',
- threading.current_thread().name)
- sys.exit(0)
-
- except Exception:
- msg = ('#TRACEBACK', format_exc())
-
- try:
- try:
- send(msg)
+ raise ke
+
+ if methodname not in exposed:
+ raise AttributeError(
+ 'method %r of %r object is not in exposed=%r' %
+ (methodname, type(obj), exposed)
+ )
+
+ function = getattr(obj, methodname)
+
+ try:
+ res = function(*args, **kwds)
+ except Exception as e:
+ msg = ('#ERROR', e)
+ else:
+ typeid = gettypeid and gettypeid.get(methodname, None)
+ if typeid:
+ rident, rexposed = self.create(conn, typeid, res)
+ token = Token(typeid, self.address, rident)
+ msg = ('#PROXY', (rexposed, token))
+ else:
+ msg = ('#RETURN', res)
+
+ except AttributeError:
+ if methodname is None:
+ msg = ('#TRACEBACK', format_exc())
+ else:
+ try:
+ fallback_func = self.fallback_mapping[methodname]
+ result = fallback_func(
+ self, conn, ident, obj, *args, **kwds
+ )
+ msg = ('#RETURN', result)
+ except Exception:
+ msg = ('#TRACEBACK', format_exc())
+
+ except EOFError:
+ util.debug('got EOF -- exiting thread serving %r',
+ threading.current_thread().name)
+ sys.exit(0)
+
+ except Exception:
+ msg = ('#TRACEBACK', format_exc())
+
+ try:
+ try:
+ send(msg)
except Exception:
- send(('#UNSERIALIZABLE', format_exc()))
- except Exception as e:
- util.info('exception in thread serving %r',
- threading.current_thread().name)
- util.info(' ... message was %r', msg)
- util.info(' ... exception was %r', e)
- conn.close()
- sys.exit(1)
-
- def fallback_getvalue(self, conn, ident, obj):
- return obj
-
- def fallback_str(self, conn, ident, obj):
- return str(obj)
-
- def fallback_repr(self, conn, ident, obj):
- return repr(obj)
-
- fallback_mapping = {
- '__str__':fallback_str,
- '__repr__':fallback_repr,
- '#GETVALUE':fallback_getvalue
- }
-
- def dummy(self, c):
- pass
-
- def debug_info(self, c):
- '''
- Return some info --- useful to spot problems with refcounting
- '''
- # Perhaps include debug info about 'c'?
- with self.mutex:
- result = []
- keys = list(self.id_to_refcount.keys())
- keys.sort()
- for ident in keys:
- if ident != '0':
- result.append(' %s: refcount=%s\n %s' %
- (ident, self.id_to_refcount[ident],
- str(self.id_to_obj[ident][0])[:75]))
- return '\n'.join(result)
-
- def number_of_objects(self, c):
- '''
- Number of shared objects
- '''
- # Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0'
- return len(self.id_to_refcount)
-
- def shutdown(self, c):
- '''
- Shutdown this process
- '''
- try:
- util.debug('manager received shutdown message')
- c.send(('#RETURN', None))
- except:
- import traceback
- traceback.print_exc()
- finally:
- self.stop_event.set()
-
+ send(('#UNSERIALIZABLE', format_exc()))
+ except Exception as e:
+ util.info('exception in thread serving %r',
+ threading.current_thread().name)
+ util.info(' ... message was %r', msg)
+ util.info(' ... exception was %r', e)
+ conn.close()
+ sys.exit(1)
+
+ def fallback_getvalue(self, conn, ident, obj):
+ return obj
+
+ def fallback_str(self, conn, ident, obj):
+ return str(obj)
+
+ def fallback_repr(self, conn, ident, obj):
+ return repr(obj)
+
+ fallback_mapping = {
+ '__str__':fallback_str,
+ '__repr__':fallback_repr,
+ '#GETVALUE':fallback_getvalue
+ }
+
+ def dummy(self, c):
+ pass
+
+ def debug_info(self, c):
+ '''
+ Return some info --- useful to spot problems with refcounting
+ '''
+ # Perhaps include debug info about 'c'?
+ with self.mutex:
+ result = []
+ keys = list(self.id_to_refcount.keys())
+ keys.sort()
+ for ident in keys:
+ if ident != '0':
+ result.append(' %s: refcount=%s\n %s' %
+ (ident, self.id_to_refcount[ident],
+ str(self.id_to_obj[ident][0])[:75]))
+ return '\n'.join(result)
+
+ def number_of_objects(self, c):
+ '''
+ Number of shared objects
+ '''
+ # Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0'
+ return len(self.id_to_refcount)
+
+ def shutdown(self, c):
+ '''
+ Shutdown this process
+ '''
+ try:
+ util.debug('manager received shutdown message')
+ c.send(('#RETURN', None))
+ except:
+ import traceback
+ traceback.print_exc()
+ finally:
+ self.stop_event.set()
+
def create(self, c, typeid, /, *args, **kwds):
- '''
- Create a new shared object and return its id
- '''
- with self.mutex:
- callable, exposed, method_to_typeid, proxytype = \
- self.registry[typeid]
-
- if callable is None:
- if kwds or (len(args) != 1):
- raise ValueError(
- "Without callable, must have one non-keyword argument")
- obj = args[0]
- else:
- obj = callable(*args, **kwds)
-
- if exposed is None:
- exposed = public_methods(obj)
- if method_to_typeid is not None:
- if not isinstance(method_to_typeid, dict):
- raise TypeError(
- "Method_to_typeid {0!r}: type {1!s}, not dict".format(
- method_to_typeid, type(method_to_typeid)))
- exposed = list(exposed) + list(method_to_typeid)
-
- ident = '%x' % id(obj) # convert to string because xmlrpclib
- # only has 32 bit signed integers
- util.debug('%r callable returned object with id %r', typeid, ident)
-
- self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid)
- if ident not in self.id_to_refcount:
- self.id_to_refcount[ident] = 0
-
- self.incref(c, ident)
- return ident, tuple(exposed)
-
- def get_methods(self, c, token):
- '''
- Return the methods of the shared object indicated by token
- '''
- return tuple(self.id_to_obj[token.id][1])
-
- def accept_connection(self, c, name):
- '''
- Spawn a new thread to serve this connection
- '''
- threading.current_thread().name = name
- c.send(('#RETURN', None))
- self.serve_client(c)
-
- def incref(self, c, ident):
- with self.mutex:
- try:
- self.id_to_refcount[ident] += 1
- except KeyError as ke:
- # If no external references exist but an internal (to the
- # manager) still does and a new external reference is created
- # from it, restore the manager's tracking of it from the
- # previously stashed internal ref.
- if ident in self.id_to_local_proxy_obj:
- self.id_to_refcount[ident] = 1
- self.id_to_obj[ident] = \
- self.id_to_local_proxy_obj[ident]
- obj, exposed, gettypeid = self.id_to_obj[ident]
- util.debug('Server re-enabled tracking & INCREF %r', ident)
- else:
- raise ke
-
- def decref(self, c, ident):
- if ident not in self.id_to_refcount and \
- ident in self.id_to_local_proxy_obj:
- util.debug('Server DECREF skipping %r', ident)
- return
-
- with self.mutex:
- if self.id_to_refcount[ident] <= 0:
- raise AssertionError(
- "Id {0!s} ({1!r}) has refcount {2:n}, not 1+".format(
- ident, self.id_to_obj[ident],
- self.id_to_refcount[ident]))
- self.id_to_refcount[ident] -= 1
- if self.id_to_refcount[ident] == 0:
- del self.id_to_refcount[ident]
-
- if ident not in self.id_to_refcount:
- # Two-step process in case the object turns out to contain other
- # proxy objects (e.g. a managed list of managed lists).
- # Otherwise, deleting self.id_to_obj[ident] would trigger the
- # deleting of the stored value (another managed object) which would
- # in turn attempt to acquire the mutex that is already held here.
- self.id_to_obj[ident] = (None, (), None) # thread-safe
- util.debug('disposing of obj with id %r', ident)
- with self.mutex:
- del self.id_to_obj[ident]
-
-
-#
-# Class to represent state of a manager
-#
-
-class State(object):
- __slots__ = ['value']
- INITIAL = 0
- STARTED = 1
- SHUTDOWN = 2
-
-#
-# Mapping from serializer name to Listener and Client types
-#
-
-listener_client = {
- 'pickle' : (connection.Listener, connection.Client),
- 'xmlrpclib' : (connection.XmlListener, connection.XmlClient)
- }
-
-#
-# Definition of BaseManager
-#
-
-class BaseManager(object):
- '''
- Base class for managers
- '''
- _registry = {}
- _Server = Server
-
- def __init__(self, address=None, authkey=None, serializer='pickle',
- ctx=None):
- if authkey is None:
- authkey = process.current_process().authkey
- self._address = address # XXX not final address if eg ('', 0)
- self._authkey = process.AuthenticationString(authkey)
- self._state = State()
- self._state.value = State.INITIAL
- self._serializer = serializer
- self._Listener, self._Client = listener_client[serializer]
- self._ctx = ctx or get_context()
-
- def get_server(self):
- '''
- Return server object with serve_forever() method and address attribute
- '''
- if self._state.value != State.INITIAL:
- if self._state.value == State.STARTED:
- raise ProcessError("Already started server")
- elif self._state.value == State.SHUTDOWN:
- raise ProcessError("Manager has shut down")
- else:
- raise ProcessError(
- "Unknown state {!r}".format(self._state.value))
- return Server(self._registry, self._address,
- self._authkey, self._serializer)
-
- def connect(self):
- '''
- Connect manager object to the server process
- '''
- Listener, Client = listener_client[self._serializer]
- conn = Client(self._address, authkey=self._authkey)
- dispatch(conn, None, 'dummy')
- self._state.value = State.STARTED
-
- def start(self, initializer=None, initargs=()):
- '''
- Spawn a server process for this manager object
- '''
- if self._state.value != State.INITIAL:
- if self._state.value == State.STARTED:
- raise ProcessError("Already started server")
- elif self._state.value == State.SHUTDOWN:
- raise ProcessError("Manager has shut down")
- else:
- raise ProcessError(
- "Unknown state {!r}".format(self._state.value))
-
- if initializer is not None and not callable(initializer):
- raise TypeError('initializer must be a callable')
-
- # pipe over which we will retrieve address of server
- reader, writer = connection.Pipe(duplex=False)
-
- # spawn process which runs a server
- self._process = self._ctx.Process(
- target=type(self)._run_server,
- args=(self._registry, self._address, self._authkey,
- self._serializer, writer, initializer, initargs),
- )
- ident = ':'.join(str(i) for i in self._process._identity)
- self._process.name = type(self).__name__ + '-' + ident
- self._process.start()
-
- # get address of server
- writer.close()
- self._address = reader.recv()
- reader.close()
-
- # register a finalizer
- self._state.value = State.STARTED
- self.shutdown = util.Finalize(
- self, type(self)._finalize_manager,
- args=(self._process, self._address, self._authkey,
- self._state, self._Client),
- exitpriority=0
- )
-
- @classmethod
- def _run_server(cls, registry, address, authkey, serializer, writer,
- initializer=None, initargs=()):
- '''
- Create a server, report its address and run it
- '''
+ '''
+ Create a new shared object and return its id
+ '''
+ with self.mutex:
+ callable, exposed, method_to_typeid, proxytype = \
+ self.registry[typeid]
+
+ if callable is None:
+ if kwds or (len(args) != 1):
+ raise ValueError(
+ "Without callable, must have one non-keyword argument")
+ obj = args[0]
+ else:
+ obj = callable(*args, **kwds)
+
+ if exposed is None:
+ exposed = public_methods(obj)
+ if method_to_typeid is not None:
+ if not isinstance(method_to_typeid, dict):
+ raise TypeError(
+ "Method_to_typeid {0!r}: type {1!s}, not dict".format(
+ method_to_typeid, type(method_to_typeid)))
+ exposed = list(exposed) + list(method_to_typeid)
+
+ ident = '%x' % id(obj) # convert to string because xmlrpclib
+ # only has 32 bit signed integers
+ util.debug('%r callable returned object with id %r', typeid, ident)
+
+ self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid)
+ if ident not in self.id_to_refcount:
+ self.id_to_refcount[ident] = 0
+
+ self.incref(c, ident)
+ return ident, tuple(exposed)
+
+ def get_methods(self, c, token):
+ '''
+ Return the methods of the shared object indicated by token
+ '''
+ return tuple(self.id_to_obj[token.id][1])
+
+ def accept_connection(self, c, name):
+ '''
+ Spawn a new thread to serve this connection
+ '''
+ threading.current_thread().name = name
+ c.send(('#RETURN', None))
+ self.serve_client(c)
+
+ def incref(self, c, ident):
+ with self.mutex:
+ try:
+ self.id_to_refcount[ident] += 1
+ except KeyError as ke:
+ # If no external references exist but an internal (to the
+ # manager) still does and a new external reference is created
+ # from it, restore the manager's tracking of it from the
+ # previously stashed internal ref.
+ if ident in self.id_to_local_proxy_obj:
+ self.id_to_refcount[ident] = 1
+ self.id_to_obj[ident] = \
+ self.id_to_local_proxy_obj[ident]
+ obj, exposed, gettypeid = self.id_to_obj[ident]
+ util.debug('Server re-enabled tracking & INCREF %r', ident)
+ else:
+ raise ke
+
+ def decref(self, c, ident):
+ if ident not in self.id_to_refcount and \
+ ident in self.id_to_local_proxy_obj:
+ util.debug('Server DECREF skipping %r', ident)
+ return
+
+ with self.mutex:
+ if self.id_to_refcount[ident] <= 0:
+ raise AssertionError(
+ "Id {0!s} ({1!r}) has refcount {2:n}, not 1+".format(
+ ident, self.id_to_obj[ident],
+ self.id_to_refcount[ident]))
+ self.id_to_refcount[ident] -= 1
+ if self.id_to_refcount[ident] == 0:
+ del self.id_to_refcount[ident]
+
+ if ident not in self.id_to_refcount:
+ # Two-step process in case the object turns out to contain other
+ # proxy objects (e.g. a managed list of managed lists).
+ # Otherwise, deleting self.id_to_obj[ident] would trigger the
+ # deleting of the stored value (another managed object) which would
+ # in turn attempt to acquire the mutex that is already held here.
+ self.id_to_obj[ident] = (None, (), None) # thread-safe
+ util.debug('disposing of obj with id %r', ident)
+ with self.mutex:
+ del self.id_to_obj[ident]
+
+
+#
+# Class to represent state of a manager
+#
+
+class State(object):
+ __slots__ = ['value']
+ INITIAL = 0
+ STARTED = 1
+ SHUTDOWN = 2
+
+#
+# Mapping from serializer name to Listener and Client types
+#
+
+listener_client = {
+ 'pickle' : (connection.Listener, connection.Client),
+ 'xmlrpclib' : (connection.XmlListener, connection.XmlClient)
+ }
+
+#
+# Definition of BaseManager
+#
+
+class BaseManager(object):
+ '''
+ Base class for managers
+ '''
+ _registry = {}
+ _Server = Server
+
+ def __init__(self, address=None, authkey=None, serializer='pickle',
+ ctx=None):
+ if authkey is None:
+ authkey = process.current_process().authkey
+ self._address = address # XXX not final address if eg ('', 0)
+ self._authkey = process.AuthenticationString(authkey)
+ self._state = State()
+ self._state.value = State.INITIAL
+ self._serializer = serializer
+ self._Listener, self._Client = listener_client[serializer]
+ self._ctx = ctx or get_context()
+
+ def get_server(self):
+ '''
+ Return server object with serve_forever() method and address attribute
+ '''
+ if self._state.value != State.INITIAL:
+ if self._state.value == State.STARTED:
+ raise ProcessError("Already started server")
+ elif self._state.value == State.SHUTDOWN:
+ raise ProcessError("Manager has shut down")
+ else:
+ raise ProcessError(
+ "Unknown state {!r}".format(self._state.value))
+ return Server(self._registry, self._address,
+ self._authkey, self._serializer)
+
+ def connect(self):
+ '''
+ Connect manager object to the server process
+ '''
+ Listener, Client = listener_client[self._serializer]
+ conn = Client(self._address, authkey=self._authkey)
+ dispatch(conn, None, 'dummy')
+ self._state.value = State.STARTED
+
+ def start(self, initializer=None, initargs=()):
+ '''
+ Spawn a server process for this manager object
+ '''
+ if self._state.value != State.INITIAL:
+ if self._state.value == State.STARTED:
+ raise ProcessError("Already started server")
+ elif self._state.value == State.SHUTDOWN:
+ raise ProcessError("Manager has shut down")
+ else:
+ raise ProcessError(
+ "Unknown state {!r}".format(self._state.value))
+
+ if initializer is not None and not callable(initializer):
+ raise TypeError('initializer must be a callable')
+
+ # pipe over which we will retrieve address of server
+ reader, writer = connection.Pipe(duplex=False)
+
+ # spawn process which runs a server
+ self._process = self._ctx.Process(
+ target=type(self)._run_server,
+ args=(self._registry, self._address, self._authkey,
+ self._serializer, writer, initializer, initargs),
+ )
+ ident = ':'.join(str(i) for i in self._process._identity)
+ self._process.name = type(self).__name__ + '-' + ident
+ self._process.start()
+
+ # get address of server
+ writer.close()
+ self._address = reader.recv()
+ reader.close()
+
+ # register a finalizer
+ self._state.value = State.STARTED
+ self.shutdown = util.Finalize(
+ self, type(self)._finalize_manager,
+ args=(self._process, self._address, self._authkey,
+ self._state, self._Client),
+ exitpriority=0
+ )
+
+ @classmethod
+ def _run_server(cls, registry, address, authkey, serializer, writer,
+ initializer=None, initargs=()):
+ '''
+ Create a server, report its address and run it
+ '''
# bpo-36368: protect server process from KeyboardInterrupt signals
signal.signal(signal.SIGINT, signal.SIG_IGN)
- if initializer is not None:
- initializer(*initargs)
-
- # create server
- server = cls._Server(registry, address, authkey, serializer)
-
- # inform parent process of the server's address
- writer.send(server.address)
- writer.close()
-
- # run the manager
- util.info('manager serving at %r', server.address)
- server.serve_forever()
-
+ if initializer is not None:
+ initializer(*initargs)
+
+ # create server
+ server = cls._Server(registry, address, authkey, serializer)
+
+ # inform parent process of the server's address
+ writer.send(server.address)
+ writer.close()
+
+ # run the manager
+ util.info('manager serving at %r', server.address)
+ server.serve_forever()
+
def _create(self, typeid, /, *args, **kwds):
- '''
- Create a new shared object; return the token and exposed tuple
- '''
- assert self._state.value == State.STARTED, 'server not yet started'
- conn = self._Client(self._address, authkey=self._authkey)
- try:
- id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
- finally:
- conn.close()
- return Token(typeid, self._address, id), exposed
-
- def join(self, timeout=None):
- '''
- Join the manager process (if it has been spawned)
- '''
- if self._process is not None:
- self._process.join(timeout)
- if not self._process.is_alive():
- self._process = None
-
- def _debug_info(self):
- '''
- Return some info about the servers shared objects and connections
- '''
- conn = self._Client(self._address, authkey=self._authkey)
- try:
- return dispatch(conn, None, 'debug_info')
- finally:
- conn.close()
-
- def _number_of_objects(self):
- '''
- Return the number of shared objects
- '''
- conn = self._Client(self._address, authkey=self._authkey)
- try:
- return dispatch(conn, None, 'number_of_objects')
- finally:
- conn.close()
-
- def __enter__(self):
- if self._state.value == State.INITIAL:
- self.start()
- if self._state.value != State.STARTED:
- if self._state.value == State.INITIAL:
- raise ProcessError("Unable to start server")
- elif self._state.value == State.SHUTDOWN:
- raise ProcessError("Manager has shut down")
- else:
- raise ProcessError(
- "Unknown state {!r}".format(self._state.value))
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.shutdown()
-
- @staticmethod
- def _finalize_manager(process, address, authkey, state, _Client):
- '''
- Shutdown the manager process; will be registered as a finalizer
- '''
- if process.is_alive():
- util.info('sending shutdown message to manager')
- try:
- conn = _Client(address, authkey=authkey)
- try:
- dispatch(conn, None, 'shutdown')
- finally:
- conn.close()
- except Exception:
- pass
-
- process.join(timeout=1.0)
- if process.is_alive():
- util.info('manager still alive')
- if hasattr(process, 'terminate'):
- util.info('trying to `terminate()` manager process')
- process.terminate()
- process.join(timeout=0.1)
- if process.is_alive():
- util.info('manager still alive after terminate')
-
- state.value = State.SHUTDOWN
- try:
- del BaseProxy._address_to_local[address]
- except KeyError:
- pass
-
- @property
- def address(self):
- return self._address
-
- @classmethod
- def register(cls, typeid, callable=None, proxytype=None, exposed=None,
- method_to_typeid=None, create_method=True):
- '''
- Register a typeid with the manager type
- '''
- if '_registry' not in cls.__dict__:
- cls._registry = cls._registry.copy()
-
- if proxytype is None:
- proxytype = AutoProxy
-
- exposed = exposed or getattr(proxytype, '_exposed_', None)
-
- method_to_typeid = method_to_typeid or \
- getattr(proxytype, '_method_to_typeid_', None)
-
- if method_to_typeid:
- for key, value in list(method_to_typeid.items()): # isinstance?
- assert type(key) is str, '%r is not a string' % key
- assert type(value) is str, '%r is not a string' % value
-
- cls._registry[typeid] = (
- callable, exposed, method_to_typeid, proxytype
- )
-
- if create_method:
+ '''
+ Create a new shared object; return the token and exposed tuple
+ '''
+ assert self._state.value == State.STARTED, 'server not yet started'
+ conn = self._Client(self._address, authkey=self._authkey)
+ try:
+ id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
+ finally:
+ conn.close()
+ return Token(typeid, self._address, id), exposed
+
+ def join(self, timeout=None):
+ '''
+ Join the manager process (if it has been spawned)
+ '''
+ if self._process is not None:
+ self._process.join(timeout)
+ if not self._process.is_alive():
+ self._process = None
+
+ def _debug_info(self):
+ '''
+ Return some info about the servers shared objects and connections
+ '''
+ conn = self._Client(self._address, authkey=self._authkey)
+ try:
+ return dispatch(conn, None, 'debug_info')
+ finally:
+ conn.close()
+
+ def _number_of_objects(self):
+ '''
+ Return the number of shared objects
+ '''
+ conn = self._Client(self._address, authkey=self._authkey)
+ try:
+ return dispatch(conn, None, 'number_of_objects')
+ finally:
+ conn.close()
+
+ def __enter__(self):
+ if self._state.value == State.INITIAL:
+ self.start()
+ if self._state.value != State.STARTED:
+ if self._state.value == State.INITIAL:
+ raise ProcessError("Unable to start server")
+ elif self._state.value == State.SHUTDOWN:
+ raise ProcessError("Manager has shut down")
+ else:
+ raise ProcessError(
+ "Unknown state {!r}".format(self._state.value))
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.shutdown()
+
+ @staticmethod
+ def _finalize_manager(process, address, authkey, state, _Client):
+ '''
+ Shutdown the manager process; will be registered as a finalizer
+ '''
+ if process.is_alive():
+ util.info('sending shutdown message to manager')
+ try:
+ conn = _Client(address, authkey=authkey)
+ try:
+ dispatch(conn, None, 'shutdown')
+ finally:
+ conn.close()
+ except Exception:
+ pass
+
+ process.join(timeout=1.0)
+ if process.is_alive():
+ util.info('manager still alive')
+ if hasattr(process, 'terminate'):
+ util.info('trying to `terminate()` manager process')
+ process.terminate()
+ process.join(timeout=0.1)
+ if process.is_alive():
+ util.info('manager still alive after terminate')
+
+ state.value = State.SHUTDOWN
+ try:
+ del BaseProxy._address_to_local[address]
+ except KeyError:
+ pass
+
+ @property
+ def address(self):
+ return self._address
+
+ @classmethod
+ def register(cls, typeid, callable=None, proxytype=None, exposed=None,
+ method_to_typeid=None, create_method=True):
+ '''
+ Register a typeid with the manager type
+ '''
+ if '_registry' not in cls.__dict__:
+ cls._registry = cls._registry.copy()
+
+ if proxytype is None:
+ proxytype = AutoProxy
+
+ exposed = exposed or getattr(proxytype, '_exposed_', None)
+
+ method_to_typeid = method_to_typeid or \
+ getattr(proxytype, '_method_to_typeid_', None)
+
+ if method_to_typeid:
+ for key, value in list(method_to_typeid.items()): # isinstance?
+ assert type(key) is str, '%r is not a string' % key
+ assert type(value) is str, '%r is not a string' % value
+
+ cls._registry[typeid] = (
+ callable, exposed, method_to_typeid, proxytype
+ )
+
+ if create_method:
def temp(self, /, *args, **kwds):
- util.debug('requesting creation of a shared %r object', typeid)
- token, exp = self._create(typeid, *args, **kwds)
- proxy = proxytype(
- token, self._serializer, manager=self,
- authkey=self._authkey, exposed=exp
- )
- conn = self._Client(token.address, authkey=self._authkey)
- dispatch(conn, None, 'decref', (token.id,))
- return proxy
- temp.__name__ = typeid
- setattr(cls, typeid, temp)
-
-#
-# Subclass of set which get cleared after a fork
-#
-
-class ProcessLocalSet(set):
- def __init__(self):
- util.register_after_fork(self, lambda obj: obj.clear())
- def __reduce__(self):
- return type(self), ()
-
-#
-# Definition of BaseProxy
-#
-
-class BaseProxy(object):
- '''
- A base for proxies of shared objects
- '''
- _address_to_local = {}
- _mutex = util.ForkAwareThreadLock()
-
- def __init__(self, token, serializer, manager=None,
- authkey=None, exposed=None, incref=True, manager_owned=False):
- with BaseProxy._mutex:
- tls_idset = BaseProxy._address_to_local.get(token.address, None)
- if tls_idset is None:
- tls_idset = util.ForkAwareLocal(), ProcessLocalSet()
- BaseProxy._address_to_local[token.address] = tls_idset
-
- # self._tls is used to record the connection used by this
- # thread to communicate with the manager at token.address
- self._tls = tls_idset[0]
-
- # self._idset is used to record the identities of all shared
- # objects for which the current process owns references and
- # which are in the manager at token.address
- self._idset = tls_idset[1]
-
- self._token = token
- self._id = self._token.id
- self._manager = manager
- self._serializer = serializer
- self._Client = listener_client[serializer][1]
-
- # Should be set to True only when a proxy object is being created
- # on the manager server; primary use case: nested proxy objects.
- # RebuildProxy detects when a proxy is being created on the manager
- # and sets this value appropriately.
- self._owned_by_manager = manager_owned
-
- if authkey is not None:
- self._authkey = process.AuthenticationString(authkey)
- elif self._manager is not None:
- self._authkey = self._manager._authkey
- else:
- self._authkey = process.current_process().authkey
-
- if incref:
- self._incref()
-
- util.register_after_fork(self, BaseProxy._after_fork)
-
- def _connect(self):
- util.debug('making connection to manager')
- name = process.current_process().name
- if threading.current_thread().name != 'MainThread':
- name += '|' + threading.current_thread().name
- conn = self._Client(self._token.address, authkey=self._authkey)
- dispatch(conn, None, 'accept_connection', (name,))
- self._tls.connection = conn
-
- def _callmethod(self, methodname, args=(), kwds={}):
- '''
+ util.debug('requesting creation of a shared %r object', typeid)
+ token, exp = self._create(typeid, *args, **kwds)
+ proxy = proxytype(
+ token, self._serializer, manager=self,
+ authkey=self._authkey, exposed=exp
+ )
+ conn = self._Client(token.address, authkey=self._authkey)
+ dispatch(conn, None, 'decref', (token.id,))
+ return proxy
+ temp.__name__ = typeid
+ setattr(cls, typeid, temp)
+
+#
+# Subclass of set which get cleared after a fork
+#
+
+class ProcessLocalSet(set):
+ def __init__(self):
+ util.register_after_fork(self, lambda obj: obj.clear())
+ def __reduce__(self):
+ return type(self), ()
+
+#
+# Definition of BaseProxy
+#
+
+class BaseProxy(object):
+ '''
+ A base for proxies of shared objects
+ '''
+ _address_to_local = {}
+ _mutex = util.ForkAwareThreadLock()
+
+ def __init__(self, token, serializer, manager=None,
+ authkey=None, exposed=None, incref=True, manager_owned=False):
+ with BaseProxy._mutex:
+ tls_idset = BaseProxy._address_to_local.get(token.address, None)
+ if tls_idset is None:
+ tls_idset = util.ForkAwareLocal(), ProcessLocalSet()
+ BaseProxy._address_to_local[token.address] = tls_idset
+
+ # self._tls is used to record the connection used by this
+ # thread to communicate with the manager at token.address
+ self._tls = tls_idset[0]
+
+ # self._idset is used to record the identities of all shared
+ # objects for which the current process owns references and
+ # which are in the manager at token.address
+ self._idset = tls_idset[1]
+
+ self._token = token
+ self._id = self._token.id
+ self._manager = manager
+ self._serializer = serializer
+ self._Client = listener_client[serializer][1]
+
+ # Should be set to True only when a proxy object is being created
+ # on the manager server; primary use case: nested proxy objects.
+ # RebuildProxy detects when a proxy is being created on the manager
+ # and sets this value appropriately.
+ self._owned_by_manager = manager_owned
+
+ if authkey is not None:
+ self._authkey = process.AuthenticationString(authkey)
+ elif self._manager is not None:
+ self._authkey = self._manager._authkey
+ else:
+ self._authkey = process.current_process().authkey
+
+ if incref:
+ self._incref()
+
+ util.register_after_fork(self, BaseProxy._after_fork)
+
+ def _connect(self):
+ util.debug('making connection to manager')
+ name = process.current_process().name
+ if threading.current_thread().name != 'MainThread':
+ name += '|' + threading.current_thread().name
+ conn = self._Client(self._token.address, authkey=self._authkey)
+ dispatch(conn, None, 'accept_connection', (name,))
+ self._tls.connection = conn
+
+ def _callmethod(self, methodname, args=(), kwds={}):
+ '''
Try to call a method of the referent and return a copy of the result
- '''
- try:
- conn = self._tls.connection
- except AttributeError:
- util.debug('thread %r does not own a connection',
- threading.current_thread().name)
- self._connect()
- conn = self._tls.connection
-
- conn.send((self._id, methodname, args, kwds))
- kind, result = conn.recv()
-
- if kind == '#RETURN':
- return result
- elif kind == '#PROXY':
- exposed, token = result
- proxytype = self._manager._registry[token.typeid][-1]
- token.address = self._token.address
- proxy = proxytype(
- token, self._serializer, manager=self._manager,
- authkey=self._authkey, exposed=exposed
- )
- conn = self._Client(token.address, authkey=self._authkey)
- dispatch(conn, None, 'decref', (token.id,))
- return proxy
- raise convert_to_error(kind, result)
-
- def _getvalue(self):
- '''
- Get a copy of the value of the referent
- '''
- return self._callmethod('#GETVALUE')
-
- def _incref(self):
- if self._owned_by_manager:
- util.debug('owned_by_manager skipped INCREF of %r', self._token.id)
- return
-
- conn = self._Client(self._token.address, authkey=self._authkey)
- dispatch(conn, None, 'incref', (self._id,))
- util.debug('INCREF %r', self._token.id)
-
- self._idset.add(self._id)
-
- state = self._manager and self._manager._state
-
- self._close = util.Finalize(
- self, BaseProxy._decref,
- args=(self._token, self._authkey, state,
- self._tls, self._idset, self._Client),
- exitpriority=10
- )
-
- @staticmethod
- def _decref(token, authkey, state, tls, idset, _Client):
- idset.discard(token.id)
-
- # check whether manager is still alive
- if state is None or state.value == State.STARTED:
- # tell manager this process no longer cares about referent
- try:
- util.debug('DECREF %r', token.id)
- conn = _Client(token.address, authkey=authkey)
- dispatch(conn, None, 'decref', (token.id,))
- except Exception as e:
- util.debug('... decref failed %s', e)
-
- else:
- util.debug('DECREF %r -- manager already shutdown', token.id)
-
- # check whether we can close this thread's connection because
- # the process owns no more references to objects for this manager
- if not idset and hasattr(tls, 'connection'):
- util.debug('thread %r has no more proxies so closing conn',
- threading.current_thread().name)
- tls.connection.close()
- del tls.connection
-
- def _after_fork(self):
- self._manager = None
- try:
- self._incref()
- except Exception as e:
- # the proxy may just be for a manager which has shutdown
- util.info('incref failed: %s' % e)
-
- def __reduce__(self):
- kwds = {}
- if get_spawning_popen() is not None:
- kwds['authkey'] = self._authkey
-
- if getattr(self, '_isauto', False):
- kwds['exposed'] = self._exposed_
- return (RebuildProxy,
- (AutoProxy, self._token, self._serializer, kwds))
- else:
- return (RebuildProxy,
- (type(self), self._token, self._serializer, kwds))
-
- def __deepcopy__(self, memo):
- return self._getvalue()
-
- def __repr__(self):
- return '<%s object, typeid %r at %#x>' % \
- (type(self).__name__, self._token.typeid, id(self))
-
- def __str__(self):
- '''
- Return representation of the referent (or a fall-back if that fails)
- '''
- try:
- return self._callmethod('__repr__')
- except Exception:
- return repr(self)[:-1] + "; '__str__()' failed>"
-
-#
-# Function used for unpickling
-#
-
-def RebuildProxy(func, token, serializer, kwds):
- '''
- Function used for unpickling proxy objects.
- '''
- server = getattr(process.current_process(), '_manager_server', None)
- if server and server.address == token.address:
- util.debug('Rebuild a proxy owned by manager, token=%r', token)
- kwds['manager_owned'] = True
- if token.id not in server.id_to_local_proxy_obj:
- server.id_to_local_proxy_obj[token.id] = \
- server.id_to_obj[token.id]
- incref = (
- kwds.pop('incref', True) and
- not getattr(process.current_process(), '_inheriting', False)
- )
- return func(token, serializer, incref=incref, **kwds)
-
-#
-# Functions to create proxies and proxy types
-#
-
-def MakeProxyType(name, exposed, _cache={}):
- '''
- Return a proxy type whose methods are given by `exposed`
- '''
- exposed = tuple(exposed)
- try:
- return _cache[(name, exposed)]
- except KeyError:
- pass
-
- dic = {}
-
- for meth in exposed:
+ '''
+ try:
+ conn = self._tls.connection
+ except AttributeError:
+ util.debug('thread %r does not own a connection',
+ threading.current_thread().name)
+ self._connect()
+ conn = self._tls.connection
+
+ conn.send((self._id, methodname, args, kwds))
+ kind, result = conn.recv()
+
+ if kind == '#RETURN':
+ return result
+ elif kind == '#PROXY':
+ exposed, token = result
+ proxytype = self._manager._registry[token.typeid][-1]
+ token.address = self._token.address
+ proxy = proxytype(
+ token, self._serializer, manager=self._manager,
+ authkey=self._authkey, exposed=exposed
+ )
+ conn = self._Client(token.address, authkey=self._authkey)
+ dispatch(conn, None, 'decref', (token.id,))
+ return proxy
+ raise convert_to_error(kind, result)
+
+ def _getvalue(self):
+ '''
+ Get a copy of the value of the referent
+ '''
+ return self._callmethod('#GETVALUE')
+
+ def _incref(self):
+ if self._owned_by_manager:
+ util.debug('owned_by_manager skipped INCREF of %r', self._token.id)
+ return
+
+ conn = self._Client(self._token.address, authkey=self._authkey)
+ dispatch(conn, None, 'incref', (self._id,))
+ util.debug('INCREF %r', self._token.id)
+
+ self._idset.add(self._id)
+
+ state = self._manager and self._manager._state
+
+ self._close = util.Finalize(
+ self, BaseProxy._decref,
+ args=(self._token, self._authkey, state,
+ self._tls, self._idset, self._Client),
+ exitpriority=10
+ )
+
+ @staticmethod
+ def _decref(token, authkey, state, tls, idset, _Client):
+ idset.discard(token.id)
+
+ # check whether manager is still alive
+ if state is None or state.value == State.STARTED:
+ # tell manager this process no longer cares about referent
+ try:
+ util.debug('DECREF %r', token.id)
+ conn = _Client(token.address, authkey=authkey)
+ dispatch(conn, None, 'decref', (token.id,))
+ except Exception as e:
+ util.debug('... decref failed %s', e)
+
+ else:
+ util.debug('DECREF %r -- manager already shutdown', token.id)
+
+ # check whether we can close this thread's connection because
+ # the process owns no more references to objects for this manager
+ if not idset and hasattr(tls, 'connection'):
+ util.debug('thread %r has no more proxies so closing conn',
+ threading.current_thread().name)
+ tls.connection.close()
+ del tls.connection
+
+ def _after_fork(self):
+ self._manager = None
+ try:
+ self._incref()
+ except Exception as e:
+ # the proxy may just be for a manager which has shutdown
+ util.info('incref failed: %s' % e)
+
+ def __reduce__(self):
+ kwds = {}
+ if get_spawning_popen() is not None:
+ kwds['authkey'] = self._authkey
+
+ if getattr(self, '_isauto', False):
+ kwds['exposed'] = self._exposed_
+ return (RebuildProxy,
+ (AutoProxy, self._token, self._serializer, kwds))
+ else:
+ return (RebuildProxy,
+ (type(self), self._token, self._serializer, kwds))
+
+ def __deepcopy__(self, memo):
+ return self._getvalue()
+
+ def __repr__(self):
+ return '<%s object, typeid %r at %#x>' % \
+ (type(self).__name__, self._token.typeid, id(self))
+
+ def __str__(self):
+ '''
+ Return representation of the referent (or a fall-back if that fails)
+ '''
+ try:
+ return self._callmethod('__repr__')
+ except Exception:
+ return repr(self)[:-1] + "; '__str__()' failed>"
+
+#
+# Function used for unpickling
+#
+
+def RebuildProxy(func, token, serializer, kwds):
+ '''
+ Function used for unpickling proxy objects.
+ '''
+ server = getattr(process.current_process(), '_manager_server', None)
+ if server and server.address == token.address:
+ util.debug('Rebuild a proxy owned by manager, token=%r', token)
+ kwds['manager_owned'] = True
+ if token.id not in server.id_to_local_proxy_obj:
+ server.id_to_local_proxy_obj[token.id] = \
+ server.id_to_obj[token.id]
+ incref = (
+ kwds.pop('incref', True) and
+ not getattr(process.current_process(), '_inheriting', False)
+ )
+ return func(token, serializer, incref=incref, **kwds)
+
+#
+# Functions to create proxies and proxy types
+#
+
+def MakeProxyType(name, exposed, _cache={}):
+ '''
+ Return a proxy type whose methods are given by `exposed`
+ '''
+ exposed = tuple(exposed)
+ try:
+ return _cache[(name, exposed)]
+ except KeyError:
+ pass
+
+ dic = {}
+
+ for meth in exposed:
exec('''def %s(self, /, *args, **kwds):
- return self._callmethod(%r, args, kwds)''' % (meth, meth), dic)
-
- ProxyType = type(name, (BaseProxy,), dic)
- ProxyType._exposed_ = exposed
- _cache[(name, exposed)] = ProxyType
- return ProxyType
-
-
-def AutoProxy(token, serializer, manager=None, authkey=None,
+ return self._callmethod(%r, args, kwds)''' % (meth, meth), dic)
+
+ ProxyType = type(name, (BaseProxy,), dic)
+ ProxyType._exposed_ = exposed
+ _cache[(name, exposed)] = ProxyType
+ return ProxyType
+
+
+def AutoProxy(token, serializer, manager=None, authkey=None,
exposed=None, incref=True, manager_owned=False):
- '''
- Return an auto-proxy for `token`
- '''
- _Client = listener_client[serializer][1]
-
- if exposed is None:
- conn = _Client(token.address, authkey=authkey)
- try:
- exposed = dispatch(conn, None, 'get_methods', (token,))
- finally:
- conn.close()
-
- if authkey is None and manager is not None:
- authkey = manager._authkey
- if authkey is None:
- authkey = process.current_process().authkey
-
- ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
- proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
+ '''
+ Return an auto-proxy for `token`
+ '''
+ _Client = listener_client[serializer][1]
+
+ if exposed is None:
+ conn = _Client(token.address, authkey=authkey)
+ try:
+ exposed = dispatch(conn, None, 'get_methods', (token,))
+ finally:
+ conn.close()
+
+ if authkey is None and manager is not None:
+ authkey = manager._authkey
+ if authkey is None:
+ authkey = process.current_process().authkey
+
+ ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
+ proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
incref=incref, manager_owned=manager_owned)
- proxy._isauto = True
- return proxy
-
-#
-# Types/callables which we will register with SyncManager
-#
-
-class Namespace(object):
+ proxy._isauto = True
+ return proxy
+
+#
+# Types/callables which we will register with SyncManager
+#
+
+class Namespace(object):
def __init__(self, /, **kwds):
- self.__dict__.update(kwds)
- def __repr__(self):
- items = list(self.__dict__.items())
- temp = []
- for name, value in items:
- if not name.startswith('_'):
- temp.append('%s=%r' % (name, value))
- temp.sort()
- return '%s(%s)' % (self.__class__.__name__, ', '.join(temp))
-
-class Value(object):
- def __init__(self, typecode, value, lock=True):
- self._typecode = typecode
- self._value = value
- def get(self):
- return self._value
- def set(self, value):
- self._value = value
- def __repr__(self):
- return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
- value = property(get, set)
-
-def Array(typecode, sequence, lock=True):
- return array.array(typecode, sequence)
-
-#
-# Proxy types used by SyncManager
-#
-
-class IteratorProxy(BaseProxy):
- _exposed_ = ('__next__', 'send', 'throw', 'close')
- def __iter__(self):
- return self
- def __next__(self, *args):
- return self._callmethod('__next__', args)
- def send(self, *args):
- return self._callmethod('send', args)
- def throw(self, *args):
- return self._callmethod('throw', args)
- def close(self, *args):
- return self._callmethod('close', args)
-
-
-class AcquirerProxy(BaseProxy):
- _exposed_ = ('acquire', 'release')
- def acquire(self, blocking=True, timeout=None):
- args = (blocking,) if timeout is None else (blocking, timeout)
- return self._callmethod('acquire', args)
- def release(self):
- return self._callmethod('release')
- def __enter__(self):
- return self._callmethod('acquire')
- def __exit__(self, exc_type, exc_val, exc_tb):
- return self._callmethod('release')
-
-
-class ConditionProxy(AcquirerProxy):
- _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all')
- def wait(self, timeout=None):
- return self._callmethod('wait', (timeout,))
- def notify(self, n=1):
- return self._callmethod('notify', (n,))
- def notify_all(self):
- return self._callmethod('notify_all')
- def wait_for(self, predicate, timeout=None):
- result = predicate()
- if result:
- return result
- if timeout is not None:
- endtime = time.monotonic() + timeout
- else:
- endtime = None
- waittime = None
- while not result:
- if endtime is not None:
- waittime = endtime - time.monotonic()
- if waittime <= 0:
- break
- self.wait(waittime)
- result = predicate()
- return result
-
-
-class EventProxy(BaseProxy):
- _exposed_ = ('is_set', 'set', 'clear', 'wait')
- def is_set(self):
- return self._callmethod('is_set')
- def set(self):
- return self._callmethod('set')
- def clear(self):
- return self._callmethod('clear')
- def wait(self, timeout=None):
- return self._callmethod('wait', (timeout,))
-
-
-class BarrierProxy(BaseProxy):
- _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset')
- def wait(self, timeout=None):
- return self._callmethod('wait', (timeout,))
- def abort(self):
- return self._callmethod('abort')
- def reset(self):
- return self._callmethod('reset')
- @property
- def parties(self):
- return self._callmethod('__getattribute__', ('parties',))
- @property
- def n_waiting(self):
- return self._callmethod('__getattribute__', ('n_waiting',))
- @property
- def broken(self):
- return self._callmethod('__getattribute__', ('broken',))
-
-
-class NamespaceProxy(BaseProxy):
- _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
- def __getattr__(self, key):
- if key[0] == '_':
- return object.__getattribute__(self, key)
- callmethod = object.__getattribute__(self, '_callmethod')
- return callmethod('__getattribute__', (key,))
- def __setattr__(self, key, value):
- if key[0] == '_':
- return object.__setattr__(self, key, value)
- callmethod = object.__getattribute__(self, '_callmethod')
- return callmethod('__setattr__', (key, value))
- def __delattr__(self, key):
- if key[0] == '_':
- return object.__delattr__(self, key)
- callmethod = object.__getattribute__(self, '_callmethod')
- return callmethod('__delattr__', (key,))
-
-
-class ValueProxy(BaseProxy):
- _exposed_ = ('get', 'set')
- def get(self):
- return self._callmethod('get')
- def set(self, value):
- return self._callmethod('set', (value,))
- value = property(get, set)
-
+ self.__dict__.update(kwds)
+ def __repr__(self):
+ items = list(self.__dict__.items())
+ temp = []
+ for name, value in items:
+ if not name.startswith('_'):
+ temp.append('%s=%r' % (name, value))
+ temp.sort()
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(temp))
+
+class Value(object):
+ def __init__(self, typecode, value, lock=True):
+ self._typecode = typecode
+ self._value = value
+ def get(self):
+ return self._value
+ def set(self, value):
+ self._value = value
+ def __repr__(self):
+ return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
+ value = property(get, set)
+
+def Array(typecode, sequence, lock=True):
+ return array.array(typecode, sequence)
+
+#
+# Proxy types used by SyncManager
+#
+
+class IteratorProxy(BaseProxy):
+ _exposed_ = ('__next__', 'send', 'throw', 'close')
+ def __iter__(self):
+ return self
+ def __next__(self, *args):
+ return self._callmethod('__next__', args)
+ def send(self, *args):
+ return self._callmethod('send', args)
+ def throw(self, *args):
+ return self._callmethod('throw', args)
+ def close(self, *args):
+ return self._callmethod('close', args)
+
+
+class AcquirerProxy(BaseProxy):
+ _exposed_ = ('acquire', 'release')
+ def acquire(self, blocking=True, timeout=None):
+ args = (blocking,) if timeout is None else (blocking, timeout)
+ return self._callmethod('acquire', args)
+ def release(self):
+ return self._callmethod('release')
+ def __enter__(self):
+ return self._callmethod('acquire')
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ return self._callmethod('release')
+
+
+class ConditionProxy(AcquirerProxy):
+ _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all')
+ def wait(self, timeout=None):
+ return self._callmethod('wait', (timeout,))
+ def notify(self, n=1):
+ return self._callmethod('notify', (n,))
+ def notify_all(self):
+ return self._callmethod('notify_all')
+ def wait_for(self, predicate, timeout=None):
+ result = predicate()
+ if result:
+ return result
+ if timeout is not None:
+ endtime = time.monotonic() + timeout
+ else:
+ endtime = None
+ waittime = None
+ while not result:
+ if endtime is not None:
+ waittime = endtime - time.monotonic()
+ if waittime <= 0:
+ break
+ self.wait(waittime)
+ result = predicate()
+ return result
+
+
+class EventProxy(BaseProxy):
+ _exposed_ = ('is_set', 'set', 'clear', 'wait')
+ def is_set(self):
+ return self._callmethod('is_set')
+ def set(self):
+ return self._callmethod('set')
+ def clear(self):
+ return self._callmethod('clear')
+ def wait(self, timeout=None):
+ return self._callmethod('wait', (timeout,))
+
+
+class BarrierProxy(BaseProxy):
+ _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset')
+ def wait(self, timeout=None):
+ return self._callmethod('wait', (timeout,))
+ def abort(self):
+ return self._callmethod('abort')
+ def reset(self):
+ return self._callmethod('reset')
+ @property
+ def parties(self):
+ return self._callmethod('__getattribute__', ('parties',))
+ @property
+ def n_waiting(self):
+ return self._callmethod('__getattribute__', ('n_waiting',))
+ @property
+ def broken(self):
+ return self._callmethod('__getattribute__', ('broken',))
+
+
+class NamespaceProxy(BaseProxy):
+ _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
+ def __getattr__(self, key):
+ if key[0] == '_':
+ return object.__getattribute__(self, key)
+ callmethod = object.__getattribute__(self, '_callmethod')
+ return callmethod('__getattribute__', (key,))
+ def __setattr__(self, key, value):
+ if key[0] == '_':
+ return object.__setattr__(self, key, value)
+ callmethod = object.__getattribute__(self, '_callmethod')
+ return callmethod('__setattr__', (key, value))
+ def __delattr__(self, key):
+ if key[0] == '_':
+ return object.__delattr__(self, key)
+ callmethod = object.__getattribute__(self, '_callmethod')
+ return callmethod('__delattr__', (key,))
+
+
+class ValueProxy(BaseProxy):
+ _exposed_ = ('get', 'set')
+ def get(self):
+ return self._callmethod('get')
+ def set(self, value):
+ return self._callmethod('set', (value,))
+ value = property(get, set)
+
__class_getitem__ = classmethod(types.GenericAlias)
+
-
-BaseListProxy = MakeProxyType('BaseListProxy', (
- '__add__', '__contains__', '__delitem__', '__getitem__', '__len__',
- '__mul__', '__reversed__', '__rmul__', '__setitem__',
- 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
- 'reverse', 'sort', '__imul__'
- ))
-class ListProxy(BaseListProxy):
- def __iadd__(self, value):
- self._callmethod('extend', (value,))
- return self
- def __imul__(self, value):
- self._callmethod('__imul__', (value,))
- return self
-
-
-DictProxy = MakeProxyType('DictProxy', (
- '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
- '__setitem__', 'clear', 'copy', 'get', 'items',
- 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
- ))
-DictProxy._method_to_typeid_ = {
- '__iter__': 'Iterator',
- }
-
-
-ArrayProxy = MakeProxyType('ArrayProxy', (
- '__len__', '__getitem__', '__setitem__'
- ))
-
-
-BasePoolProxy = MakeProxyType('PoolProxy', (
- 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
- 'map', 'map_async', 'starmap', 'starmap_async', 'terminate',
- ))
-BasePoolProxy._method_to_typeid_ = {
- 'apply_async': 'AsyncResult',
- 'map_async': 'AsyncResult',
- 'starmap_async': 'AsyncResult',
- 'imap': 'Iterator',
- 'imap_unordered': 'Iterator'
- }
-class PoolProxy(BasePoolProxy):
- def __enter__(self):
- return self
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.terminate()
-
-#
-# Definition of SyncManager
-#
-
-class SyncManager(BaseManager):
- '''
- Subclass of `BaseManager` which supports a number of shared object types.
-
- The types registered are those intended for the synchronization
- of threads, plus `dict`, `list` and `Namespace`.
-
- The `multiprocessing.Manager()` function creates started instances of
- this class.
- '''
-
-SyncManager.register('Queue', queue.Queue)
-SyncManager.register('JoinableQueue', queue.Queue)
-SyncManager.register('Event', threading.Event, EventProxy)
-SyncManager.register('Lock', threading.Lock, AcquirerProxy)
-SyncManager.register('RLock', threading.RLock, AcquirerProxy)
-SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
-SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
- AcquirerProxy)
-SyncManager.register('Condition', threading.Condition, ConditionProxy)
-SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
-SyncManager.register('Pool', pool.Pool, PoolProxy)
-SyncManager.register('list', list, ListProxy)
-SyncManager.register('dict', dict, DictProxy)
-SyncManager.register('Value', Value, ValueProxy)
-SyncManager.register('Array', Array, ArrayProxy)
-SyncManager.register('Namespace', Namespace, NamespaceProxy)
-
-# types returned by methods of PoolProxy
-SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
-SyncManager.register('AsyncResult', create_method=False)
+BaseListProxy = MakeProxyType('BaseListProxy', (
+ '__add__', '__contains__', '__delitem__', '__getitem__', '__len__',
+ '__mul__', '__reversed__', '__rmul__', '__setitem__',
+ 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
+ 'reverse', 'sort', '__imul__'
+ ))
+class ListProxy(BaseListProxy):
+ def __iadd__(self, value):
+ self._callmethod('extend', (value,))
+ return self
+ def __imul__(self, value):
+ self._callmethod('__imul__', (value,))
+ return self
+
+
+DictProxy = MakeProxyType('DictProxy', (
+ '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
+ '__setitem__', 'clear', 'copy', 'get', 'items',
+ 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
+ ))
+DictProxy._method_to_typeid_ = {
+ '__iter__': 'Iterator',
+ }
+
+
+ArrayProxy = MakeProxyType('ArrayProxy', (
+ '__len__', '__getitem__', '__setitem__'
+ ))
+
+
+BasePoolProxy = MakeProxyType('PoolProxy', (
+ 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
+ 'map', 'map_async', 'starmap', 'starmap_async', 'terminate',
+ ))
+BasePoolProxy._method_to_typeid_ = {
+ 'apply_async': 'AsyncResult',
+ 'map_async': 'AsyncResult',
+ 'starmap_async': 'AsyncResult',
+ 'imap': 'Iterator',
+ 'imap_unordered': 'Iterator'
+ }
+class PoolProxy(BasePoolProxy):
+ def __enter__(self):
+ return self
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.terminate()
+
+#
+# Definition of SyncManager
+#
+
+class SyncManager(BaseManager):
+ '''
+ Subclass of `BaseManager` which supports a number of shared object types.
+
+ The types registered are those intended for the synchronization
+ of threads, plus `dict`, `list` and `Namespace`.
+
+ The `multiprocessing.Manager()` function creates started instances of
+ this class.
+ '''
+
+SyncManager.register('Queue', queue.Queue)
+SyncManager.register('JoinableQueue', queue.Queue)
+SyncManager.register('Event', threading.Event, EventProxy)
+SyncManager.register('Lock', threading.Lock, AcquirerProxy)
+SyncManager.register('RLock', threading.RLock, AcquirerProxy)
+SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
+SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
+ AcquirerProxy)
+SyncManager.register('Condition', threading.Condition, ConditionProxy)
+SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
+SyncManager.register('Pool', pool.Pool, PoolProxy)
+SyncManager.register('list', list, ListProxy)
+SyncManager.register('dict', dict, DictProxy)
+SyncManager.register('Value', Value, ValueProxy)
+SyncManager.register('Array', Array, ArrayProxy)
+SyncManager.register('Namespace', Namespace, NamespaceProxy)
+
+# types returned by methods of PoolProxy
+SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
+SyncManager.register('AsyncResult', create_method=False)
#
# Definition of SharedMemoryManager and SharedMemoryServer
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/pool.py b/contrib/tools/python3/src/Lib/multiprocessing/pool.py
index bbe05a550c3..fd54580a369 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/pool.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/pool.py
@@ -1,152 +1,152 @@
-#
-# Module providing the `Pool` class for managing a process pool
-#
-# multiprocessing/pool.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-__all__ = ['Pool', 'ThreadPool']
-
-#
-# Imports
-#
-
+#
+# Module providing the `Pool` class for managing a process pool
+#
+# multiprocessing/pool.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = ['Pool', 'ThreadPool']
+
+#
+# Imports
+#
+
import collections
-import itertools
-import os
+import itertools
+import os
import queue
import threading
-import time
-import traceback
+import time
+import traceback
import types
import warnings
-
-# If threading is available then ThreadPool should be provided. Therefore
-# we avoid top-level imports which are liable to fail on some systems.
-from . import util
-from . import get_context, TimeoutError
+
+# If threading is available then ThreadPool should be provided. Therefore
+# we avoid top-level imports which are liable to fail on some systems.
+from . import util
+from . import get_context, TimeoutError
from .connection import wait
-
-#
-# Constants representing the state of a pool
-#
-
+
+#
+# Constants representing the state of a pool
+#
+
INIT = "INIT"
RUN = "RUN"
CLOSE = "CLOSE"
TERMINATE = "TERMINATE"
-
-#
-# Miscellaneous
-#
-
-job_counter = itertools.count()
-
-def mapstar(args):
- return list(map(*args))
-
-def starmapstar(args):
- return list(itertools.starmap(args[0], args[1]))
-
-#
-# Hack to embed stringification of remote traceback in local traceback
-#
-
-class RemoteTraceback(Exception):
- def __init__(self, tb):
- self.tb = tb
- def __str__(self):
- return self.tb
-
-class ExceptionWithTraceback:
- def __init__(self, exc, tb):
- tb = traceback.format_exception(type(exc), exc, tb)
- tb = ''.join(tb)
- self.exc = exc
- self.tb = '\n"""\n%s"""' % tb
- def __reduce__(self):
- return rebuild_exc, (self.exc, self.tb)
-
-def rebuild_exc(exc, tb):
- exc.__cause__ = RemoteTraceback(tb)
- return exc
-
-#
-# Code run by worker processes
-#
-
-class MaybeEncodingError(Exception):
- """Wraps possible unpickleable errors, so they can be
- safely sent through the socket."""
-
- def __init__(self, exc, value):
- self.exc = repr(exc)
- self.value = repr(value)
- super(MaybeEncodingError, self).__init__(self.exc, self.value)
-
- def __str__(self):
- return "Error sending result: '%s'. Reason: '%s'" % (self.value,
- self.exc)
-
- def __repr__(self):
- return "<%s: %s>" % (self.__class__.__name__, self)
-
-
-def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None,
- wrap_exception=False):
- if (maxtasks is not None) and not (isinstance(maxtasks, int)
- and maxtasks >= 1):
- raise AssertionError("Maxtasks {!r} is not valid".format(maxtasks))
- put = outqueue.put
- get = inqueue.get
- if hasattr(inqueue, '_writer'):
- inqueue._writer.close()
- outqueue._reader.close()
-
- if initializer is not None:
- initializer(*initargs)
-
- completed = 0
- while maxtasks is None or (maxtasks and completed < maxtasks):
- try:
- task = get()
- except (EOFError, OSError):
- util.debug('worker got EOFError or OSError -- exiting')
- break
-
- if task is None:
- util.debug('worker got sentinel -- exiting')
- break
-
- job, i, func, args, kwds = task
- try:
- result = (True, func(*args, **kwds))
- except Exception as e:
- if wrap_exception and func is not _helper_reraises_exception:
- e = ExceptionWithTraceback(e, e.__traceback__)
- result = (False, e)
- try:
- put((job, i, result))
- except Exception as e:
- wrapped = MaybeEncodingError(e, result[1])
- util.debug("Possible encoding error while sending result: %s" % (
- wrapped))
- put((job, i, (False, wrapped)))
-
- task = job = result = func = args = kwds = None
- completed += 1
- util.debug('worker exiting after %d tasks' % completed)
-
-def _helper_reraises_exception(ex):
- 'Pickle-able helper function for use by _guarded_task_generation.'
- raise ex
-
-#
-# Class representing a process pool
-#
-
+
+#
+# Miscellaneous
+#
+
+job_counter = itertools.count()
+
+def mapstar(args):
+ return list(map(*args))
+
+def starmapstar(args):
+ return list(itertools.starmap(args[0], args[1]))
+
+#
+# Hack to embed stringification of remote traceback in local traceback
+#
+
+class RemoteTraceback(Exception):
+ def __init__(self, tb):
+ self.tb = tb
+ def __str__(self):
+ return self.tb
+
+class ExceptionWithTraceback:
+ def __init__(self, exc, tb):
+ tb = traceback.format_exception(type(exc), exc, tb)
+ tb = ''.join(tb)
+ self.exc = exc
+ self.tb = '\n"""\n%s"""' % tb
+ def __reduce__(self):
+ return rebuild_exc, (self.exc, self.tb)
+
+def rebuild_exc(exc, tb):
+ exc.__cause__ = RemoteTraceback(tb)
+ return exc
+
+#
+# Code run by worker processes
+#
+
+class MaybeEncodingError(Exception):
+ """Wraps possible unpickleable errors, so they can be
+ safely sent through the socket."""
+
+ def __init__(self, exc, value):
+ self.exc = repr(exc)
+ self.value = repr(value)
+ super(MaybeEncodingError, self).__init__(self.exc, self.value)
+
+ def __str__(self):
+ return "Error sending result: '%s'. Reason: '%s'" % (self.value,
+ self.exc)
+
+ def __repr__(self):
+ return "<%s: %s>" % (self.__class__.__name__, self)
+
+
+def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None,
+ wrap_exception=False):
+ if (maxtasks is not None) and not (isinstance(maxtasks, int)
+ and maxtasks >= 1):
+ raise AssertionError("Maxtasks {!r} is not valid".format(maxtasks))
+ put = outqueue.put
+ get = inqueue.get
+ if hasattr(inqueue, '_writer'):
+ inqueue._writer.close()
+ outqueue._reader.close()
+
+ if initializer is not None:
+ initializer(*initargs)
+
+ completed = 0
+ while maxtasks is None or (maxtasks and completed < maxtasks):
+ try:
+ task = get()
+ except (EOFError, OSError):
+ util.debug('worker got EOFError or OSError -- exiting')
+ break
+
+ if task is None:
+ util.debug('worker got sentinel -- exiting')
+ break
+
+ job, i, func, args, kwds = task
+ try:
+ result = (True, func(*args, **kwds))
+ except Exception as e:
+ if wrap_exception and func is not _helper_reraises_exception:
+ e = ExceptionWithTraceback(e, e.__traceback__)
+ result = (False, e)
+ try:
+ put((job, i, result))
+ except Exception as e:
+ wrapped = MaybeEncodingError(e, result[1])
+ util.debug("Possible encoding error while sending result: %s" % (
+ wrapped))
+ put((job, i, (False, wrapped)))
+
+ task = job = result = func = args = kwds = None
+ completed += 1
+ util.debug('worker exiting after %d tasks' % completed)
+
+def _helper_reraises_exception(ex):
+ 'Pickle-able helper function for use by _guarded_task_generation.'
+ raise ex
+
+#
+# Class representing a process pool
+#
+
class _PoolCache(dict):
"""
Class that implements a cache for the Pool class that will notify
@@ -170,44 +170,44 @@ class _PoolCache(dict):
if not self:
self.notifier.put(None)
-class Pool(object):
- '''
- Class which supports an async version of applying functions to arguments.
- '''
- _wrap_exception = True
-
+class Pool(object):
+ '''
+ Class which supports an async version of applying functions to arguments.
+ '''
+ _wrap_exception = True
+
@staticmethod
def Process(ctx, *args, **kwds):
return ctx.Process(*args, **kwds)
-
- def __init__(self, processes=None, initializer=None, initargs=(),
- maxtasksperchild=None, context=None):
+
+ def __init__(self, processes=None, initializer=None, initargs=(),
+ maxtasksperchild=None, context=None):
# Attributes initialized early to make sure that they exist in
# __del__() if __init__() raises an exception
self._pool = []
self._state = INIT
- self._ctx = context or get_context()
- self._setup_queues()
- self._taskqueue = queue.SimpleQueue()
+ self._ctx = context or get_context()
+ self._setup_queues()
+ self._taskqueue = queue.SimpleQueue()
# The _change_notifier queue exist to wake up self._handle_workers()
# when the cache (self._cache) is empty or when there is a change in
# the _state variable of the thread that runs _handle_workers.
self._change_notifier = self._ctx.SimpleQueue()
self._cache = _PoolCache(notifier=self._change_notifier)
- self._maxtasksperchild = maxtasksperchild
- self._initializer = initializer
- self._initargs = initargs
-
- if processes is None:
- processes = os.cpu_count() or 1
- if processes < 1:
- raise ValueError("Number of processes must be at least 1")
-
- if initializer is not None and not callable(initializer):
- raise TypeError('initializer must be a callable')
-
- self._processes = processes
+ self._maxtasksperchild = maxtasksperchild
+ self._initializer = initializer
+ self._initargs = initargs
+
+ if processes is None:
+ processes = os.cpu_count() or 1
+ if processes < 1:
+ raise ValueError("Number of processes must be at least 1")
+
+ if initializer is not None and not callable(initializer):
+ raise TypeError('initializer must be a callable')
+
+ self._processes = processes
try:
self._repopulate_pool()
except Exception:
@@ -217,47 +217,47 @@ class Pool(object):
for p in self._pool:
p.join()
raise
-
+
sentinels = self._get_sentinels()
- self._worker_handler = threading.Thread(
- target=Pool._handle_workers,
+ self._worker_handler = threading.Thread(
+ target=Pool._handle_workers,
args=(self._cache, self._taskqueue, self._ctx, self.Process,
self._processes, self._pool, self._inqueue, self._outqueue,
self._initializer, self._initargs, self._maxtasksperchild,
self._wrap_exception, sentinels, self._change_notifier)
- )
- self._worker_handler.daemon = True
- self._worker_handler._state = RUN
- self._worker_handler.start()
-
-
- self._task_handler = threading.Thread(
- target=Pool._handle_tasks,
- args=(self._taskqueue, self._quick_put, self._outqueue,
- self._pool, self._cache)
- )
- self._task_handler.daemon = True
- self._task_handler._state = RUN
- self._task_handler.start()
-
- self._result_handler = threading.Thread(
- target=Pool._handle_results,
- args=(self._outqueue, self._quick_get, self._cache)
- )
- self._result_handler.daemon = True
- self._result_handler._state = RUN
- self._result_handler.start()
-
- self._terminate = util.Finalize(
- self, self._terminate_pool,
- args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
+ )
+ self._worker_handler.daemon = True
+ self._worker_handler._state = RUN
+ self._worker_handler.start()
+
+
+ self._task_handler = threading.Thread(
+ target=Pool._handle_tasks,
+ args=(self._taskqueue, self._quick_put, self._outqueue,
+ self._pool, self._cache)
+ )
+ self._task_handler.daemon = True
+ self._task_handler._state = RUN
+ self._task_handler.start()
+
+ self._result_handler = threading.Thread(
+ target=Pool._handle_results,
+ args=(self._outqueue, self._quick_get, self._cache)
+ )
+ self._result_handler.daemon = True
+ self._result_handler._state = RUN
+ self._result_handler.start()
+
+ self._terminate = util.Finalize(
+ self, self._terminate_pool,
+ args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
self._change_notifier, self._worker_handler, self._task_handler,
- self._result_handler, self._cache),
- exitpriority=15
- )
+ self._result_handler, self._cache),
+ exitpriority=15
+ )
self._state = RUN
-
+
# Copy globals as function locals to make sure that they are available
# during Python shutdown when the Pool is destroyed.
def __del__(self, _warn=warnings.warn, RUN=RUN):
@@ -285,21 +285,21 @@ class Pool(object):
@staticmethod
def _join_exited_workers(pool):
- """Cleanup after any worker processes which have exited due to reaching
- their specified lifetime. Returns True if any workers were cleaned up.
- """
- cleaned = False
+ """Cleanup after any worker processes which have exited due to reaching
+ their specified lifetime. Returns True if any workers were cleaned up.
+ """
+ cleaned = False
for i in reversed(range(len(pool))):
worker = pool[i]
- if worker.exitcode is not None:
- # worker exited
- util.debug('cleaning up worker %d' % i)
- worker.join()
- cleaned = True
+ if worker.exitcode is not None:
+ # worker exited
+ util.debug('cleaning up worker %d' % i)
+ worker.join()
+ cleaned = True
del pool[i]
- return cleaned
-
- def _repopulate_pool(self):
+ return cleaned
+
+ def _repopulate_pool(self):
return self._repopulate_pool_static(self._ctx, self.Process,
self._processes,
self._pool, self._inqueue,
@@ -312,189 +312,189 @@ class Pool(object):
def _repopulate_pool_static(ctx, Process, processes, pool, inqueue,
outqueue, initializer, initargs,
maxtasksperchild, wrap_exception):
- """Bring the number of pool processes up to the specified number,
- for use after reaping workers which have exited.
- """
+ """Bring the number of pool processes up to the specified number,
+ for use after reaping workers which have exited.
+ """
for i in range(processes - len(pool)):
w = Process(ctx, target=worker,
args=(inqueue, outqueue,
initializer,
initargs, maxtasksperchild,
wrap_exception))
- w.name = w.name.replace('Process', 'PoolWorker')
- w.daemon = True
- w.start()
+ w.name = w.name.replace('Process', 'PoolWorker')
+ w.daemon = True
+ w.start()
pool.append(w)
- util.debug('added worker')
-
+ util.debug('added worker')
+
@staticmethod
def _maintain_pool(ctx, Process, processes, pool, inqueue, outqueue,
initializer, initargs, maxtasksperchild,
wrap_exception):
- """Clean up any exited workers and start replacements for them.
- """
+ """Clean up any exited workers and start replacements for them.
+ """
if Pool._join_exited_workers(pool):
Pool._repopulate_pool_static(ctx, Process, processes, pool,
inqueue, outqueue, initializer,
initargs, maxtasksperchild,
wrap_exception)
-
- def _setup_queues(self):
- self._inqueue = self._ctx.SimpleQueue()
- self._outqueue = self._ctx.SimpleQueue()
- self._quick_put = self._inqueue._writer.send
- self._quick_get = self._outqueue._reader.recv
-
+
+ def _setup_queues(self):
+ self._inqueue = self._ctx.SimpleQueue()
+ self._outqueue = self._ctx.SimpleQueue()
+ self._quick_put = self._inqueue._writer.send
+ self._quick_get = self._outqueue._reader.recv
+
def _check_running(self):
if self._state != RUN:
raise ValueError("Pool not running")
- def apply(self, func, args=(), kwds={}):
- '''
- Equivalent of `func(*args, **kwds)`.
- Pool must be running.
- '''
- return self.apply_async(func, args, kwds).get()
-
- def map(self, func, iterable, chunksize=None):
- '''
- Apply `func` to each element in `iterable`, collecting the results
- in a list that is returned.
- '''
- return self._map_async(func, iterable, mapstar, chunksize).get()
-
- def starmap(self, func, iterable, chunksize=None):
- '''
- Like `map()` method but the elements of the `iterable` are expected to
- be iterables as well and will be unpacked as arguments. Hence
- `func` and (a, b) becomes func(a, b).
- '''
- return self._map_async(func, iterable, starmapstar, chunksize).get()
-
- def starmap_async(self, func, iterable, chunksize=None, callback=None,
- error_callback=None):
- '''
- Asynchronous version of `starmap()` method.
- '''
- return self._map_async(func, iterable, starmapstar, chunksize,
- callback, error_callback)
-
- def _guarded_task_generation(self, result_job, func, iterable):
- '''Provides a generator of tasks for imap and imap_unordered with
- appropriate handling for iterables which throw exceptions during
- iteration.'''
- try:
- i = -1
- for i, x in enumerate(iterable):
- yield (result_job, i, func, (x,), {})
- except Exception as e:
- yield (result_job, i+1, _helper_reraises_exception, (e,), {})
-
- def imap(self, func, iterable, chunksize=1):
- '''
- Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
- '''
+ def apply(self, func, args=(), kwds={}):
+ '''
+ Equivalent of `func(*args, **kwds)`.
+ Pool must be running.
+ '''
+ return self.apply_async(func, args, kwds).get()
+
+ def map(self, func, iterable, chunksize=None):
+ '''
+ Apply `func` to each element in `iterable`, collecting the results
+ in a list that is returned.
+ '''
+ return self._map_async(func, iterable, mapstar, chunksize).get()
+
+ def starmap(self, func, iterable, chunksize=None):
+ '''
+ Like `map()` method but the elements of the `iterable` are expected to
+ be iterables as well and will be unpacked as arguments. Hence
+ `func` and (a, b) becomes func(a, b).
+ '''
+ return self._map_async(func, iterable, starmapstar, chunksize).get()
+
+ def starmap_async(self, func, iterable, chunksize=None, callback=None,
+ error_callback=None):
+ '''
+ Asynchronous version of `starmap()` method.
+ '''
+ return self._map_async(func, iterable, starmapstar, chunksize,
+ callback, error_callback)
+
+ def _guarded_task_generation(self, result_job, func, iterable):
+ '''Provides a generator of tasks for imap and imap_unordered with
+ appropriate handling for iterables which throw exceptions during
+ iteration.'''
+ try:
+ i = -1
+ for i, x in enumerate(iterable):
+ yield (result_job, i, func, (x,), {})
+ except Exception as e:
+ yield (result_job, i+1, _helper_reraises_exception, (e,), {})
+
+ def imap(self, func, iterable, chunksize=1):
+ '''
+ Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
+ '''
self._check_running()
- if chunksize == 1:
+ if chunksize == 1:
result = IMapIterator(self)
- self._taskqueue.put(
- (
- self._guarded_task_generation(result._job, func, iterable),
- result._set_length
- ))
- return result
- else:
- if chunksize < 1:
- raise ValueError(
- "Chunksize must be 1+, not {0:n}".format(
- chunksize))
- task_batches = Pool._get_tasks(func, iterable, chunksize)
+ self._taskqueue.put(
+ (
+ self._guarded_task_generation(result._job, func, iterable),
+ result._set_length
+ ))
+ return result
+ else:
+ if chunksize < 1:
+ raise ValueError(
+ "Chunksize must be 1+, not {0:n}".format(
+ chunksize))
+ task_batches = Pool._get_tasks(func, iterable, chunksize)
result = IMapIterator(self)
- self._taskqueue.put(
- (
- self._guarded_task_generation(result._job,
- mapstar,
- task_batches),
- result._set_length
- ))
- return (item for chunk in result for item in chunk)
-
- def imap_unordered(self, func, iterable, chunksize=1):
- '''
- Like `imap()` method but ordering of results is arbitrary.
- '''
+ self._taskqueue.put(
+ (
+ self._guarded_task_generation(result._job,
+ mapstar,
+ task_batches),
+ result._set_length
+ ))
+ return (item for chunk in result for item in chunk)
+
+ def imap_unordered(self, func, iterable, chunksize=1):
+ '''
+ Like `imap()` method but ordering of results is arbitrary.
+ '''
self._check_running()
- if chunksize == 1:
+ if chunksize == 1:
result = IMapUnorderedIterator(self)
- self._taskqueue.put(
- (
- self._guarded_task_generation(result._job, func, iterable),
- result._set_length
- ))
- return result
- else:
- if chunksize < 1:
- raise ValueError(
- "Chunksize must be 1+, not {0!r}".format(chunksize))
- task_batches = Pool._get_tasks(func, iterable, chunksize)
+ self._taskqueue.put(
+ (
+ self._guarded_task_generation(result._job, func, iterable),
+ result._set_length
+ ))
+ return result
+ else:
+ if chunksize < 1:
+ raise ValueError(
+ "Chunksize must be 1+, not {0!r}".format(chunksize))
+ task_batches = Pool._get_tasks(func, iterable, chunksize)
result = IMapUnorderedIterator(self)
- self._taskqueue.put(
- (
- self._guarded_task_generation(result._job,
- mapstar,
- task_batches),
- result._set_length
- ))
- return (item for chunk in result for item in chunk)
-
- def apply_async(self, func, args=(), kwds={}, callback=None,
- error_callback=None):
- '''
- Asynchronous version of `apply()` method.
- '''
+ self._taskqueue.put(
+ (
+ self._guarded_task_generation(result._job,
+ mapstar,
+ task_batches),
+ result._set_length
+ ))
+ return (item for chunk in result for item in chunk)
+
+ def apply_async(self, func, args=(), kwds={}, callback=None,
+ error_callback=None):
+ '''
+ Asynchronous version of `apply()` method.
+ '''
self._check_running()
result = ApplyResult(self, callback, error_callback)
- self._taskqueue.put(([(result._job, 0, func, args, kwds)], None))
- return result
-
- def map_async(self, func, iterable, chunksize=None, callback=None,
- error_callback=None):
- '''
- Asynchronous version of `map()` method.
- '''
- return self._map_async(func, iterable, mapstar, chunksize, callback,
- error_callback)
-
- def _map_async(self, func, iterable, mapper, chunksize=None, callback=None,
- error_callback=None):
- '''
- Helper function to implement map, starmap and their async counterparts.
- '''
+ self._taskqueue.put(([(result._job, 0, func, args, kwds)], None))
+ return result
+
+ def map_async(self, func, iterable, chunksize=None, callback=None,
+ error_callback=None):
+ '''
+ Asynchronous version of `map()` method.
+ '''
+ return self._map_async(func, iterable, mapstar, chunksize, callback,
+ error_callback)
+
+ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None,
+ error_callback=None):
+ '''
+ Helper function to implement map, starmap and their async counterparts.
+ '''
self._check_running()
- if not hasattr(iterable, '__len__'):
- iterable = list(iterable)
-
- if chunksize is None:
- chunksize, extra = divmod(len(iterable), len(self._pool) * 4)
- if extra:
- chunksize += 1
- if len(iterable) == 0:
- chunksize = 0
-
- task_batches = Pool._get_tasks(func, iterable, chunksize)
+ if not hasattr(iterable, '__len__'):
+ iterable = list(iterable)
+
+ if chunksize is None:
+ chunksize, extra = divmod(len(iterable), len(self._pool) * 4)
+ if extra:
+ chunksize += 1
+ if len(iterable) == 0:
+ chunksize = 0
+
+ task_batches = Pool._get_tasks(func, iterable, chunksize)
result = MapResult(self, chunksize, len(iterable), callback,
- error_callback=error_callback)
- self._taskqueue.put(
- (
- self._guarded_task_generation(result._job,
- mapper,
- task_batches),
- None
- )
- )
- return result
-
- @staticmethod
+ error_callback=error_callback)
+ self._taskqueue.put(
+ (
+ self._guarded_task_generation(result._job,
+ mapper,
+ task_batches),
+ None
+ )
+ )
+ return result
+
+ @staticmethod
def _wait_for_updates(sentinels, change_notifier, timeout=None):
wait(sentinels, timeout=timeout)
while not change_notifier.empty():
@@ -505,10 +505,10 @@ class Pool(object):
pool, inqueue, outqueue, initializer, initargs,
maxtasksperchild, wrap_exception, sentinels,
change_notifier):
- thread = threading.current_thread()
-
- # Keep maintaining workers until the cache gets drained, unless the pool
- # is terminated.
+ thread = threading.current_thread()
+
+ # Keep maintaining workers until the cache gets drained, unless the pool
+ # is terminated.
while thread._state == RUN or (cache and thread._state != TERMINATE):
cls._maintain_pool(ctx, Process, processes, pool, inqueue,
outqueue, initializer, initargs,
@@ -517,438 +517,438 @@ class Pool(object):
current_sentinels = [*cls._get_worker_sentinels(pool), *sentinels]
cls._wait_for_updates(current_sentinels, change_notifier)
- # send sentinel to stop workers
+ # send sentinel to stop workers
taskqueue.put(None)
- util.debug('worker handler exiting')
-
- @staticmethod
- def _handle_tasks(taskqueue, put, outqueue, pool, cache):
- thread = threading.current_thread()
-
- for taskseq, set_length in iter(taskqueue.get, None):
- task = None
- try:
- # iterating taskseq cannot fail
- for task in taskseq:
+ util.debug('worker handler exiting')
+
+ @staticmethod
+ def _handle_tasks(taskqueue, put, outqueue, pool, cache):
+ thread = threading.current_thread()
+
+ for taskseq, set_length in iter(taskqueue.get, None):
+ task = None
+ try:
+ # iterating taskseq cannot fail
+ for task in taskseq:
if thread._state != RUN:
- util.debug('task handler found thread._state != RUN')
- break
- try:
- put(task)
- except Exception as e:
- job, idx = task[:2]
- try:
- cache[job]._set(idx, (False, e))
- except KeyError:
- pass
- else:
- if set_length:
- util.debug('doing set_length()')
- idx = task[1] if task else -1
- set_length(idx + 1)
- continue
- break
- finally:
- task = taskseq = job = None
- else:
- util.debug('task handler got sentinel')
-
- try:
- # tell result handler to finish when cache is empty
- util.debug('task handler sending sentinel to result handler')
- outqueue.put(None)
-
- # tell workers there is no more work
- util.debug('task handler sending sentinel to workers')
- for p in pool:
- put(None)
- except OSError:
- util.debug('task handler got OSError when sending sentinels')
-
- util.debug('task handler exiting')
-
- @staticmethod
- def _handle_results(outqueue, get, cache):
- thread = threading.current_thread()
-
- while 1:
- try:
- task = get()
- except (OSError, EOFError):
- util.debug('result handler got EOFError/OSError -- exiting')
- return
-
+ util.debug('task handler found thread._state != RUN')
+ break
+ try:
+ put(task)
+ except Exception as e:
+ job, idx = task[:2]
+ try:
+ cache[job]._set(idx, (False, e))
+ except KeyError:
+ pass
+ else:
+ if set_length:
+ util.debug('doing set_length()')
+ idx = task[1] if task else -1
+ set_length(idx + 1)
+ continue
+ break
+ finally:
+ task = taskseq = job = None
+ else:
+ util.debug('task handler got sentinel')
+
+ try:
+ # tell result handler to finish when cache is empty
+ util.debug('task handler sending sentinel to result handler')
+ outqueue.put(None)
+
+ # tell workers there is no more work
+ util.debug('task handler sending sentinel to workers')
+ for p in pool:
+ put(None)
+ except OSError:
+ util.debug('task handler got OSError when sending sentinels')
+
+ util.debug('task handler exiting')
+
+ @staticmethod
+ def _handle_results(outqueue, get, cache):
+ thread = threading.current_thread()
+
+ while 1:
+ try:
+ task = get()
+ except (OSError, EOFError):
+ util.debug('result handler got EOFError/OSError -- exiting')
+ return
+
if thread._state != RUN:
- assert thread._state == TERMINATE, "Thread not in TERMINATE"
- util.debug('result handler found thread._state=TERMINATE')
- break
-
- if task is None:
- util.debug('result handler got sentinel')
- break
-
- job, i, obj = task
- try:
- cache[job]._set(i, obj)
- except KeyError:
- pass
- task = job = obj = None
-
- while cache and thread._state != TERMINATE:
- try:
- task = get()
- except (OSError, EOFError):
- util.debug('result handler got EOFError/OSError -- exiting')
- return
-
- if task is None:
- util.debug('result handler ignoring extra sentinel')
- continue
- job, i, obj = task
- try:
- cache[job]._set(i, obj)
- except KeyError:
- pass
- task = job = obj = None
-
- if hasattr(outqueue, '_reader'):
- util.debug('ensuring that outqueue is not full')
- # If we don't make room available in outqueue then
- # attempts to add the sentinel (None) to outqueue may
- # block. There is guaranteed to be no more than 2 sentinels.
- try:
- for i in range(10):
- if not outqueue._reader.poll():
- break
- get()
- except (OSError, EOFError):
- pass
-
- util.debug('result handler exiting: len(cache)=%s, thread._state=%s',
- len(cache), thread._state)
-
- @staticmethod
- def _get_tasks(func, it, size):
- it = iter(it)
- while 1:
- x = tuple(itertools.islice(it, size))
- if not x:
- return
- yield (func, x)
-
- def __reduce__(self):
- raise NotImplementedError(
- 'pool objects cannot be passed between processes or pickled'
- )
-
- def close(self):
- util.debug('closing pool')
- if self._state == RUN:
- self._state = CLOSE
- self._worker_handler._state = CLOSE
+ assert thread._state == TERMINATE, "Thread not in TERMINATE"
+ util.debug('result handler found thread._state=TERMINATE')
+ break
+
+ if task is None:
+ util.debug('result handler got sentinel')
+ break
+
+ job, i, obj = task
+ try:
+ cache[job]._set(i, obj)
+ except KeyError:
+ pass
+ task = job = obj = None
+
+ while cache and thread._state != TERMINATE:
+ try:
+ task = get()
+ except (OSError, EOFError):
+ util.debug('result handler got EOFError/OSError -- exiting')
+ return
+
+ if task is None:
+ util.debug('result handler ignoring extra sentinel')
+ continue
+ job, i, obj = task
+ try:
+ cache[job]._set(i, obj)
+ except KeyError:
+ pass
+ task = job = obj = None
+
+ if hasattr(outqueue, '_reader'):
+ util.debug('ensuring that outqueue is not full')
+ # If we don't make room available in outqueue then
+ # attempts to add the sentinel (None) to outqueue may
+ # block. There is guaranteed to be no more than 2 sentinels.
+ try:
+ for i in range(10):
+ if not outqueue._reader.poll():
+ break
+ get()
+ except (OSError, EOFError):
+ pass
+
+ util.debug('result handler exiting: len(cache)=%s, thread._state=%s',
+ len(cache), thread._state)
+
+ @staticmethod
+ def _get_tasks(func, it, size):
+ it = iter(it)
+ while 1:
+ x = tuple(itertools.islice(it, size))
+ if not x:
+ return
+ yield (func, x)
+
+ def __reduce__(self):
+ raise NotImplementedError(
+ 'pool objects cannot be passed between processes or pickled'
+ )
+
+ def close(self):
+ util.debug('closing pool')
+ if self._state == RUN:
+ self._state = CLOSE
+ self._worker_handler._state = CLOSE
self._change_notifier.put(None)
-
- def terminate(self):
- util.debug('terminating pool')
- self._state = TERMINATE
- self._terminate()
-
- def join(self):
- util.debug('joining pool')
- if self._state == RUN:
- raise ValueError("Pool is still running")
- elif self._state not in (CLOSE, TERMINATE):
- raise ValueError("In unknown state")
- self._worker_handler.join()
- self._task_handler.join()
- self._result_handler.join()
- for p in self._pool:
- p.join()
-
- @staticmethod
- def _help_stuff_finish(inqueue, task_handler, size):
- # task_handler may be blocked trying to put items on inqueue
- util.debug('removing tasks from inqueue until task handler finished')
- inqueue._rlock.acquire()
- while task_handler.is_alive() and inqueue._reader.poll():
- inqueue._reader.recv()
- time.sleep(0)
-
- @classmethod
+
+ def terminate(self):
+ util.debug('terminating pool')
+ self._state = TERMINATE
+ self._terminate()
+
+ def join(self):
+ util.debug('joining pool')
+ if self._state == RUN:
+ raise ValueError("Pool is still running")
+ elif self._state not in (CLOSE, TERMINATE):
+ raise ValueError("In unknown state")
+ self._worker_handler.join()
+ self._task_handler.join()
+ self._result_handler.join()
+ for p in self._pool:
+ p.join()
+
+ @staticmethod
+ def _help_stuff_finish(inqueue, task_handler, size):
+ # task_handler may be blocked trying to put items on inqueue
+ util.debug('removing tasks from inqueue until task handler finished')
+ inqueue._rlock.acquire()
+ while task_handler.is_alive() and inqueue._reader.poll():
+ inqueue._reader.recv()
+ time.sleep(0)
+
+ @classmethod
def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier,
- worker_handler, task_handler, result_handler, cache):
- # this is guaranteed to only be called once
- util.debug('finalizing pool')
-
+ worker_handler, task_handler, result_handler, cache):
+ # this is guaranteed to only be called once
+ util.debug('finalizing pool')
+
# Notify that the worker_handler state has been changed so the
# _handle_workers loop can be unblocked (and exited) in order to
# send the finalization sentinel all the workers.
- worker_handler._state = TERMINATE
+ worker_handler._state = TERMINATE
change_notifier.put(None)
- task_handler._state = TERMINATE
-
- util.debug('helping task handler/workers to finish')
- cls._help_stuff_finish(inqueue, task_handler, len(pool))
-
- if (not result_handler.is_alive()) and (len(cache) != 0):
- raise AssertionError(
- "Cannot have cache with result_hander not alive")
-
- result_handler._state = TERMINATE
+ task_handler._state = TERMINATE
+
+ util.debug('helping task handler/workers to finish')
+ cls._help_stuff_finish(inqueue, task_handler, len(pool))
+
+ if (not result_handler.is_alive()) and (len(cache) != 0):
+ raise AssertionError(
+ "Cannot have cache with result_hander not alive")
+
+ result_handler._state = TERMINATE
change_notifier.put(None)
- outqueue.put(None) # sentinel
-
- # We must wait for the worker handler to exit before terminating
- # workers because we don't want workers to be restarted behind our back.
- util.debug('joining worker handler')
- if threading.current_thread() is not worker_handler:
- worker_handler.join()
-
- # Terminate workers which haven't already finished.
- if pool and hasattr(pool[0], 'terminate'):
- util.debug('terminating workers')
- for p in pool:
- if p.exitcode is None:
- p.terminate()
-
- util.debug('joining task handler')
- if threading.current_thread() is not task_handler:
- task_handler.join()
-
- util.debug('joining result handler')
- if threading.current_thread() is not result_handler:
- result_handler.join()
-
- if pool and hasattr(pool[0], 'terminate'):
- util.debug('joining pool workers')
- for p in pool:
- if p.is_alive():
- # worker has not yet exited
- util.debug('cleaning up worker %d' % p.pid)
- p.join()
-
- def __enter__(self):
+ outqueue.put(None) # sentinel
+
+ # We must wait for the worker handler to exit before terminating
+ # workers because we don't want workers to be restarted behind our back.
+ util.debug('joining worker handler')
+ if threading.current_thread() is not worker_handler:
+ worker_handler.join()
+
+ # Terminate workers which haven't already finished.
+ if pool and hasattr(pool[0], 'terminate'):
+ util.debug('terminating workers')
+ for p in pool:
+ if p.exitcode is None:
+ p.terminate()
+
+ util.debug('joining task handler')
+ if threading.current_thread() is not task_handler:
+ task_handler.join()
+
+ util.debug('joining result handler')
+ if threading.current_thread() is not result_handler:
+ result_handler.join()
+
+ if pool and hasattr(pool[0], 'terminate'):
+ util.debug('joining pool workers')
+ for p in pool:
+ if p.is_alive():
+ # worker has not yet exited
+ util.debug('cleaning up worker %d' % p.pid)
+ p.join()
+
+ def __enter__(self):
self._check_running()
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.terminate()
-
-#
-# Class whose instances are returned by `Pool.apply_async()`
-#
-
-class ApplyResult(object):
-
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.terminate()
+
+#
+# Class whose instances are returned by `Pool.apply_async()`
+#
+
+class ApplyResult(object):
+
def __init__(self, pool, callback, error_callback):
self._pool = pool
- self._event = threading.Event()
- self._job = next(job_counter)
+ self._event = threading.Event()
+ self._job = next(job_counter)
self._cache = pool._cache
- self._callback = callback
- self._error_callback = error_callback
+ self._callback = callback
+ self._error_callback = error_callback
self._cache[self._job] = self
-
- def ready(self):
- return self._event.is_set()
-
- def successful(self):
- if not self.ready():
- raise ValueError("{0!r} not ready".format(self))
- return self._success
-
- def wait(self, timeout=None):
- self._event.wait(timeout)
-
- def get(self, timeout=None):
- self.wait(timeout)
- if not self.ready():
- raise TimeoutError
- if self._success:
- return self._value
- else:
- raise self._value
-
- def _set(self, i, obj):
- self._success, self._value = obj
- if self._callback and self._success:
- self._callback(self._value)
- if self._error_callback and not self._success:
- self._error_callback(self._value)
- self._event.set()
- del self._cache[self._job]
+
+ def ready(self):
+ return self._event.is_set()
+
+ def successful(self):
+ if not self.ready():
+ raise ValueError("{0!r} not ready".format(self))
+ return self._success
+
+ def wait(self, timeout=None):
+ self._event.wait(timeout)
+
+ def get(self, timeout=None):
+ self.wait(timeout)
+ if not self.ready():
+ raise TimeoutError
+ if self._success:
+ return self._value
+ else:
+ raise self._value
+
+ def _set(self, i, obj):
+ self._success, self._value = obj
+ if self._callback and self._success:
+ self._callback(self._value)
+ if self._error_callback and not self._success:
+ self._error_callback(self._value)
+ self._event.set()
+ del self._cache[self._job]
self._pool = None
-
+
__class_getitem__ = classmethod(types.GenericAlias)
-AsyncResult = ApplyResult # create alias -- see #17805
-
-#
-# Class whose instances are returned by `Pool.map_async()`
-#
-
-class MapResult(ApplyResult):
-
+AsyncResult = ApplyResult # create alias -- see #17805
+
+#
+# Class whose instances are returned by `Pool.map_async()`
+#
+
+class MapResult(ApplyResult):
+
def __init__(self, pool, chunksize, length, callback, error_callback):
ApplyResult.__init__(self, pool, callback,
- error_callback=error_callback)
- self._success = True
- self._value = [None] * length
- self._chunksize = chunksize
- if chunksize <= 0:
- self._number_left = 0
- self._event.set()
+ error_callback=error_callback)
+ self._success = True
+ self._value = [None] * length
+ self._chunksize = chunksize
+ if chunksize <= 0:
+ self._number_left = 0
+ self._event.set()
del self._cache[self._job]
- else:
- self._number_left = length//chunksize + bool(length % chunksize)
-
- def _set(self, i, success_result):
- self._number_left -= 1
- success, result = success_result
- if success and self._success:
- self._value[i*self._chunksize:(i+1)*self._chunksize] = result
- if self._number_left == 0:
- if self._callback:
- self._callback(self._value)
- del self._cache[self._job]
- self._event.set()
+ else:
+ self._number_left = length//chunksize + bool(length % chunksize)
+
+ def _set(self, i, success_result):
+ self._number_left -= 1
+ success, result = success_result
+ if success and self._success:
+ self._value[i*self._chunksize:(i+1)*self._chunksize] = result
+ if self._number_left == 0:
+ if self._callback:
+ self._callback(self._value)
+ del self._cache[self._job]
+ self._event.set()
self._pool = None
- else:
- if not success and self._success:
- # only store first exception
- self._success = False
- self._value = result
- if self._number_left == 0:
- # only consider the result ready once all jobs are done
- if self._error_callback:
- self._error_callback(self._value)
- del self._cache[self._job]
- self._event.set()
+ else:
+ if not success and self._success:
+ # only store first exception
+ self._success = False
+ self._value = result
+ if self._number_left == 0:
+ # only consider the result ready once all jobs are done
+ if self._error_callback:
+ self._error_callback(self._value)
+ del self._cache[self._job]
+ self._event.set()
self._pool = None
-
-#
-# Class whose instances are returned by `Pool.imap()`
-#
-
-class IMapIterator(object):
-
+
+#
+# Class whose instances are returned by `Pool.imap()`
+#
+
+class IMapIterator(object):
+
def __init__(self, pool):
self._pool = pool
- self._cond = threading.Condition(threading.Lock())
- self._job = next(job_counter)
+ self._cond = threading.Condition(threading.Lock())
+ self._job = next(job_counter)
self._cache = pool._cache
- self._items = collections.deque()
- self._index = 0
- self._length = None
- self._unsorted = {}
+ self._items = collections.deque()
+ self._index = 0
+ self._length = None
+ self._unsorted = {}
self._cache[self._job] = self
-
- def __iter__(self):
- return self
-
- def next(self, timeout=None):
- with self._cond:
- try:
- item = self._items.popleft()
- except IndexError:
- if self._index == self._length:
+
+ def __iter__(self):
+ return self
+
+ def next(self, timeout=None):
+ with self._cond:
+ try:
+ item = self._items.popleft()
+ except IndexError:
+ if self._index == self._length:
self._pool = None
- raise StopIteration from None
- self._cond.wait(timeout)
- try:
- item = self._items.popleft()
- except IndexError:
- if self._index == self._length:
+ raise StopIteration from None
+ self._cond.wait(timeout)
+ try:
+ item = self._items.popleft()
+ except IndexError:
+ if self._index == self._length:
self._pool = None
- raise StopIteration from None
- raise TimeoutError from None
-
- success, value = item
- if success:
- return value
- raise value
-
- __next__ = next # XXX
-
- def _set(self, i, obj):
- with self._cond:
- if self._index == i:
- self._items.append(obj)
- self._index += 1
- while self._index in self._unsorted:
- obj = self._unsorted.pop(self._index)
- self._items.append(obj)
- self._index += 1
- self._cond.notify()
- else:
- self._unsorted[i] = obj
-
- if self._index == self._length:
- del self._cache[self._job]
+ raise StopIteration from None
+ raise TimeoutError from None
+
+ success, value = item
+ if success:
+ return value
+ raise value
+
+ __next__ = next # XXX
+
+ def _set(self, i, obj):
+ with self._cond:
+ if self._index == i:
+ self._items.append(obj)
+ self._index += 1
+ while self._index in self._unsorted:
+ obj = self._unsorted.pop(self._index)
+ self._items.append(obj)
+ self._index += 1
+ self._cond.notify()
+ else:
+ self._unsorted[i] = obj
+
+ if self._index == self._length:
+ del self._cache[self._job]
self._pool = None
-
- def _set_length(self, length):
- with self._cond:
- self._length = length
- if self._index == self._length:
- self._cond.notify()
- del self._cache[self._job]
+
+ def _set_length(self, length):
+ with self._cond:
+ self._length = length
+ if self._index == self._length:
+ self._cond.notify()
+ del self._cache[self._job]
self._pool = None
-
-#
-# Class whose instances are returned by `Pool.imap_unordered()`
-#
-
-class IMapUnorderedIterator(IMapIterator):
-
- def _set(self, i, obj):
- with self._cond:
- self._items.append(obj)
- self._index += 1
- self._cond.notify()
- if self._index == self._length:
- del self._cache[self._job]
+
+#
+# Class whose instances are returned by `Pool.imap_unordered()`
+#
+
+class IMapUnorderedIterator(IMapIterator):
+
+ def _set(self, i, obj):
+ with self._cond:
+ self._items.append(obj)
+ self._index += 1
+ self._cond.notify()
+ if self._index == self._length:
+ del self._cache[self._job]
self._pool = None
-
-#
-#
-#
-
-class ThreadPool(Pool):
- _wrap_exception = False
-
- @staticmethod
+
+#
+#
+#
+
+class ThreadPool(Pool):
+ _wrap_exception = False
+
+ @staticmethod
def Process(ctx, *args, **kwds):
- from .dummy import Process
- return Process(*args, **kwds)
-
- def __init__(self, processes=None, initializer=None, initargs=()):
- Pool.__init__(self, processes, initializer, initargs)
-
- def _setup_queues(self):
- self._inqueue = queue.SimpleQueue()
- self._outqueue = queue.SimpleQueue()
- self._quick_put = self._inqueue.put
- self._quick_get = self._outqueue.get
-
+ from .dummy import Process
+ return Process(*args, **kwds)
+
+ def __init__(self, processes=None, initializer=None, initargs=()):
+ Pool.__init__(self, processes, initializer, initargs)
+
+ def _setup_queues(self):
+ self._inqueue = queue.SimpleQueue()
+ self._outqueue = queue.SimpleQueue()
+ self._quick_put = self._inqueue.put
+ self._quick_get = self._outqueue.get
+
def _get_sentinels(self):
return [self._change_notifier._reader]
- @staticmethod
+ @staticmethod
def _get_worker_sentinels(workers):
return []
@staticmethod
- def _help_stuff_finish(inqueue, task_handler, size):
- # drain inqueue, and put sentinels at its head to make workers finish
- try:
- while True:
- inqueue.get(block=False)
- except queue.Empty:
- pass
- for i in range(size):
- inqueue.put(None)
+ def _help_stuff_finish(inqueue, task_handler, size):
+ # drain inqueue, and put sentinels at its head to make workers finish
+ try:
+ while True:
+ inqueue.get(block=False)
+ except queue.Empty:
+ pass
+ for i in range(size):
+ inqueue.put(None)
def _wait_for_updates(self, sentinels, change_notifier, timeout):
time.sleep(timeout)
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/popen_fork.py b/contrib/tools/python3/src/Lib/multiprocessing/popen_fork.py
index 625981cf476..87675cedbe9 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/popen_fork.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/popen_fork.py
@@ -1,83 +1,83 @@
-import os
-import signal
-
-from . import util
-
-__all__ = ['Popen']
-
-#
-# Start child process using fork
-#
-
-class Popen(object):
- method = 'fork'
-
- def __init__(self, process_obj):
- util._flush_std_streams()
- self.returncode = None
- self.finalizer = None
- self._launch(process_obj)
-
- def duplicate_for_child(self, fd):
- return fd
-
- def poll(self, flag=os.WNOHANG):
- if self.returncode is None:
- try:
- pid, sts = os.waitpid(self.pid, flag)
+import os
+import signal
+
+from . import util
+
+__all__ = ['Popen']
+
+#
+# Start child process using fork
+#
+
+class Popen(object):
+ method = 'fork'
+
+ def __init__(self, process_obj):
+ util._flush_std_streams()
+ self.returncode = None
+ self.finalizer = None
+ self._launch(process_obj)
+
+ def duplicate_for_child(self, fd):
+ return fd
+
+ def poll(self, flag=os.WNOHANG):
+ if self.returncode is None:
+ try:
+ pid, sts = os.waitpid(self.pid, flag)
except OSError:
- # Child process not yet created. See #1731717
- # e.errno == errno.ECHILD == 10
- return None
- if pid == self.pid:
+ # Child process not yet created. See #1731717
+ # e.errno == errno.ECHILD == 10
+ return None
+ if pid == self.pid:
self.returncode = os.waitstatus_to_exitcode(sts)
- return self.returncode
-
- def wait(self, timeout=None):
- if self.returncode is None:
- if timeout is not None:
- from multiprocessing.connection import wait
- if not wait([self.sentinel], timeout):
- return None
- # This shouldn't block if wait() returned successfully.
- return self.poll(os.WNOHANG if timeout == 0.0 else 0)
- return self.returncode
-
- def _send_signal(self, sig):
- if self.returncode is None:
- try:
- os.kill(self.pid, sig)
- except ProcessLookupError:
- pass
- except OSError:
- if self.wait(timeout=0.1) is None:
- raise
-
- def terminate(self):
- self._send_signal(signal.SIGTERM)
-
- def kill(self):
- self._send_signal(signal.SIGKILL)
-
- def _launch(self, process_obj):
- code = 1
- parent_r, child_w = os.pipe()
+ return self.returncode
+
+ def wait(self, timeout=None):
+ if self.returncode is None:
+ if timeout is not None:
+ from multiprocessing.connection import wait
+ if not wait([self.sentinel], timeout):
+ return None
+ # This shouldn't block if wait() returned successfully.
+ return self.poll(os.WNOHANG if timeout == 0.0 else 0)
+ return self.returncode
+
+ def _send_signal(self, sig):
+ if self.returncode is None:
+ try:
+ os.kill(self.pid, sig)
+ except ProcessLookupError:
+ pass
+ except OSError:
+ if self.wait(timeout=0.1) is None:
+ raise
+
+ def terminate(self):
+ self._send_signal(signal.SIGTERM)
+
+ def kill(self):
+ self._send_signal(signal.SIGKILL)
+
+ def _launch(self, process_obj):
+ code = 1
+ parent_r, child_w = os.pipe()
child_r, parent_w = os.pipe()
- self.pid = os.fork()
- if self.pid == 0:
- try:
- os.close(parent_r)
+ self.pid = os.fork()
+ if self.pid == 0:
+ try:
+ os.close(parent_r)
os.close(parent_w)
code = process_obj._bootstrap(parent_sentinel=child_r)
- finally:
- os._exit(code)
- else:
- os.close(child_w)
+ finally:
+ os._exit(code)
+ else:
+ os.close(child_w)
os.close(child_r)
self.finalizer = util.Finalize(self, util.close_fds,
(parent_r, parent_w,))
- self.sentinel = parent_r
-
- def close(self):
- if self.finalizer is not None:
- self.finalizer()
+ self.sentinel = parent_r
+
+ def close(self):
+ if self.finalizer is not None:
+ self.finalizer()
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/popen_forkserver.py b/contrib/tools/python3/src/Lib/multiprocessing/popen_forkserver.py
index a56eb9bf110..45505f9f9b7 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/popen_forkserver.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/popen_forkserver.py
@@ -1,74 +1,74 @@
-import io
-import os
-
-from .context import reduction, set_spawning_popen
-if not reduction.HAVE_SEND_HANDLE:
- raise ImportError('No support for sending fds between processes')
-from . import forkserver
-from . import popen_fork
-from . import spawn
-from . import util
-
-
-__all__ = ['Popen']
-
-#
-# Wrapper for an fd used while launching a process
-#
-
-class _DupFd(object):
- def __init__(self, ind):
- self.ind = ind
- def detach(self):
- return forkserver.get_inherited_fds()[self.ind]
-
-#
-# Start child process using a server process
-#
-
-class Popen(popen_fork.Popen):
- method = 'forkserver'
- DupFd = _DupFd
-
- def __init__(self, process_obj):
- self._fds = []
- super().__init__(process_obj)
-
- def duplicate_for_child(self, fd):
- self._fds.append(fd)
- return len(self._fds) - 1
-
- def _launch(self, process_obj):
- prep_data = spawn.get_preparation_data(process_obj._name)
- buf = io.BytesIO()
- set_spawning_popen(self)
- try:
- reduction.dump(prep_data, buf)
- reduction.dump(process_obj, buf)
- finally:
- set_spawning_popen(None)
-
- self.sentinel, w = forkserver.connect_to_new_process(self._fds)
+import io
+import os
+
+from .context import reduction, set_spawning_popen
+if not reduction.HAVE_SEND_HANDLE:
+ raise ImportError('No support for sending fds between processes')
+from . import forkserver
+from . import popen_fork
+from . import spawn
+from . import util
+
+
+__all__ = ['Popen']
+
+#
+# Wrapper for an fd used while launching a process
+#
+
+class _DupFd(object):
+ def __init__(self, ind):
+ self.ind = ind
+ def detach(self):
+ return forkserver.get_inherited_fds()[self.ind]
+
+#
+# Start child process using a server process
+#
+
+class Popen(popen_fork.Popen):
+ method = 'forkserver'
+ DupFd = _DupFd
+
+ def __init__(self, process_obj):
+ self._fds = []
+ super().__init__(process_obj)
+
+ def duplicate_for_child(self, fd):
+ self._fds.append(fd)
+ return len(self._fds) - 1
+
+ def _launch(self, process_obj):
+ prep_data = spawn.get_preparation_data(process_obj._name)
+ buf = io.BytesIO()
+ set_spawning_popen(self)
+ try:
+ reduction.dump(prep_data, buf)
+ reduction.dump(process_obj, buf)
+ finally:
+ set_spawning_popen(None)
+
+ self.sentinel, w = forkserver.connect_to_new_process(self._fds)
# Keep a duplicate of the data pipe's write end as a sentinel of the
# parent process used by the child process.
_parent_w = os.dup(w)
self.finalizer = util.Finalize(self, util.close_fds,
(_parent_w, self.sentinel))
- with open(w, 'wb', closefd=True) as f:
- f.write(buf.getbuffer())
- self.pid = forkserver.read_signed(self.sentinel)
-
- def poll(self, flag=os.WNOHANG):
- if self.returncode is None:
- from multiprocessing.connection import wait
- timeout = 0 if flag == os.WNOHANG else None
- if not wait([self.sentinel], timeout):
- return None
- try:
- self.returncode = forkserver.read_signed(self.sentinel)
- except (OSError, EOFError):
- # This should not happen usually, but perhaps the forkserver
- # process itself got killed
- self.returncode = 255
-
- return self.returncode
+ with open(w, 'wb', closefd=True) as f:
+ f.write(buf.getbuffer())
+ self.pid = forkserver.read_signed(self.sentinel)
+
+ def poll(self, flag=os.WNOHANG):
+ if self.returncode is None:
+ from multiprocessing.connection import wait
+ timeout = 0 if flag == os.WNOHANG else None
+ if not wait([self.sentinel], timeout):
+ return None
+ try:
+ self.returncode = forkserver.read_signed(self.sentinel)
+ except (OSError, EOFError):
+ # This should not happen usually, but perhaps the forkserver
+ # process itself got killed
+ self.returncode = 255
+
+ return self.returncode
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_posix.py b/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_posix.py
index 24b8634523e..f50145d5456 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_posix.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_posix.py
@@ -1,72 +1,72 @@
-import io
-import os
-
-from .context import reduction, set_spawning_popen
-from . import popen_fork
-from . import spawn
-from . import util
-
-__all__ = ['Popen']
-
-
-#
-# Wrapper for an fd used while launching a process
-#
-
-class _DupFd(object):
- def __init__(self, fd):
- self.fd = fd
- def detach(self):
- return self.fd
-
-#
-# Start child process using a fresh interpreter
-#
-
-class Popen(popen_fork.Popen):
- method = 'spawn'
- DupFd = _DupFd
-
- def __init__(self, process_obj):
- self._fds = []
- super().__init__(process_obj)
-
- def duplicate_for_child(self, fd):
- self._fds.append(fd)
- return fd
-
- def _launch(self, process_obj):
+import io
+import os
+
+from .context import reduction, set_spawning_popen
+from . import popen_fork
+from . import spawn
+from . import util
+
+__all__ = ['Popen']
+
+
+#
+# Wrapper for an fd used while launching a process
+#
+
+class _DupFd(object):
+ def __init__(self, fd):
+ self.fd = fd
+ def detach(self):
+ return self.fd
+
+#
+# Start child process using a fresh interpreter
+#
+
+class Popen(popen_fork.Popen):
+ method = 'spawn'
+ DupFd = _DupFd
+
+ def __init__(self, process_obj):
+ self._fds = []
+ super().__init__(process_obj)
+
+ def duplicate_for_child(self, fd):
+ self._fds.append(fd)
+ return fd
+
+ def _launch(self, process_obj):
from . import resource_tracker
tracker_fd = resource_tracker.getfd()
- self._fds.append(tracker_fd)
- prep_data = spawn.get_preparation_data(process_obj._name)
- fp = io.BytesIO()
- set_spawning_popen(self)
- try:
- reduction.dump(prep_data, fp)
- reduction.dump(process_obj, fp)
- finally:
- set_spawning_popen(None)
-
- parent_r = child_w = child_r = parent_w = None
- try:
- parent_r, child_w = os.pipe()
- child_r, parent_w = os.pipe()
- cmd = spawn.get_command_line(tracker_fd=tracker_fd,
- pipe_handle=child_r)
- self._fds.extend([child_r, child_w])
- self.pid = util.spawnv_passfds(spawn.get_executable(),
- cmd, self._fds)
- self.sentinel = parent_r
- with open(parent_w, 'wb', closefd=False) as f:
- f.write(fp.getbuffer())
- finally:
+ self._fds.append(tracker_fd)
+ prep_data = spawn.get_preparation_data(process_obj._name)
+ fp = io.BytesIO()
+ set_spawning_popen(self)
+ try:
+ reduction.dump(prep_data, fp)
+ reduction.dump(process_obj, fp)
+ finally:
+ set_spawning_popen(None)
+
+ parent_r = child_w = child_r = parent_w = None
+ try:
+ parent_r, child_w = os.pipe()
+ child_r, parent_w = os.pipe()
+ cmd = spawn.get_command_line(tracker_fd=tracker_fd,
+ pipe_handle=child_r)
+ self._fds.extend([child_r, child_w])
+ self.pid = util.spawnv_passfds(spawn.get_executable(),
+ cmd, self._fds)
+ self.sentinel = parent_r
+ with open(parent_w, 'wb', closefd=False) as f:
+ f.write(fp.getbuffer())
+ finally:
fds_to_close = []
for fd in (parent_r, parent_w):
- if fd is not None:
+ if fd is not None:
fds_to_close.append(fd)
self.finalizer = util.Finalize(self, util.close_fds, fds_to_close)
for fd in (child_r, child_w):
if fd is not None:
- os.close(fd)
+ os.close(fd)
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_win32.py b/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_win32.py
index 27fe0642906..c710c8f29cc 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_win32.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/popen_spawn_win32.py
@@ -1,132 +1,132 @@
-import os
-import msvcrt
-import signal
-import sys
-import _winapi
-
-from .context import reduction, get_spawning_popen, set_spawning_popen
-from . import spawn
-from . import util
-
-__all__ = ['Popen']
-
-#
-#
-#
-
-TERMINATE = 0x10000
-WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
-WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
-
-
-def _path_eq(p1, p2):
- return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
-
+import os
+import msvcrt
+import signal
+import sys
+import _winapi
+
+from .context import reduction, get_spawning_popen, set_spawning_popen
+from . import spawn
+from . import util
+
+__all__ = ['Popen']
+
+#
+#
+#
+
+TERMINATE = 0x10000
+WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
+WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
+
+
+def _path_eq(p1, p2):
+ return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
+
WINENV = not _path_eq(sys.executable, sys._base_executable)
-
-
-def _close_handles(*handles):
- for handle in handles:
- _winapi.CloseHandle(handle)
-
-
-#
-# We define a Popen class similar to the one from subprocess, but
-# whose constructor takes a process object as its argument.
-#
-
-class Popen(object):
- '''
- Start a subprocess to run the code of a process object
- '''
- method = 'spawn'
-
- def __init__(self, process_obj):
- prep_data = spawn.get_preparation_data(process_obj._name)
-
+
+
+def _close_handles(*handles):
+ for handle in handles:
+ _winapi.CloseHandle(handle)
+
+
+#
+# We define a Popen class similar to the one from subprocess, but
+# whose constructor takes a process object as its argument.
+#
+
+class Popen(object):
+ '''
+ Start a subprocess to run the code of a process object
+ '''
+ method = 'spawn'
+
+ def __init__(self, process_obj):
+ prep_data = spawn.get_preparation_data(process_obj._name)
+
# read end of pipe will be duplicated by the child process
- # -- see spawn_main() in spawn.py.
+ # -- see spawn_main() in spawn.py.
#
# bpo-33929: Previously, the read end of pipe was "stolen" by the child
# process, but it leaked a handle if the child process had been
# terminated before it could steal the handle from the parent process.
- rhandle, whandle = _winapi.CreatePipe(None, 0)
- wfd = msvcrt.open_osfhandle(whandle, 0)
- cmd = spawn.get_command_line(parent_pid=os.getpid(),
- pipe_handle=rhandle)
- cmd = ' '.join('"%s"' % x for x in cmd)
-
- python_exe = spawn.get_executable()
-
- # bpo-35797: When running in a venv, we bypass the redirect
- # executor and launch our base Python.
- if WINENV and _path_eq(python_exe, sys.executable):
- python_exe = sys._base_executable
- env = os.environ.copy()
- env["__PYVENV_LAUNCHER__"] = sys.executable
- else:
- env = os.environ.copy()
- env['Y_PYTHON_ENTRY_POINT'] = ':main'
-
- with open(wfd, 'wb', closefd=True) as to_child:
- # start process
- try:
- hp, ht, pid, tid = _winapi.CreateProcess(
- python_exe, cmd,
+ rhandle, whandle = _winapi.CreatePipe(None, 0)
+ wfd = msvcrt.open_osfhandle(whandle, 0)
+ cmd = spawn.get_command_line(parent_pid=os.getpid(),
+ pipe_handle=rhandle)
+ cmd = ' '.join('"%s"' % x for x in cmd)
+
+ python_exe = spawn.get_executable()
+
+ # bpo-35797: When running in a venv, we bypass the redirect
+ # executor and launch our base Python.
+ if WINENV and _path_eq(python_exe, sys.executable):
+ python_exe = sys._base_executable
+ env = os.environ.copy()
+ env["__PYVENV_LAUNCHER__"] = sys.executable
+ else:
+ env = os.environ.copy()
+ env['Y_PYTHON_ENTRY_POINT'] = ':main'
+
+ with open(wfd, 'wb', closefd=True) as to_child:
+ # start process
+ try:
+ hp, ht, pid, tid = _winapi.CreateProcess(
+ python_exe, cmd,
None, None, False, 0, env, None, None)
- _winapi.CloseHandle(ht)
- except:
- _winapi.CloseHandle(rhandle)
- raise
-
- # set attributes of self
- self.pid = pid
- self.returncode = None
- self._handle = hp
- self.sentinel = int(hp)
+ _winapi.CloseHandle(ht)
+ except:
+ _winapi.CloseHandle(rhandle)
+ raise
+
+ # set attributes of self
+ self.pid = pid
+ self.returncode = None
+ self._handle = hp
+ self.sentinel = int(hp)
self.finalizer = util.Finalize(self, _close_handles,
(self.sentinel, int(rhandle)))
-
- # send information to child
- set_spawning_popen(self)
- try:
- reduction.dump(prep_data, to_child)
- reduction.dump(process_obj, to_child)
- finally:
- set_spawning_popen(None)
-
- def duplicate_for_child(self, handle):
- assert self is get_spawning_popen()
- return reduction.duplicate(handle, self.sentinel)
-
- def wait(self, timeout=None):
- if self.returncode is None:
- if timeout is None:
- msecs = _winapi.INFINITE
- else:
- msecs = max(0, int(timeout * 1000 + 0.5))
-
- res = _winapi.WaitForSingleObject(int(self._handle), msecs)
- if res == _winapi.WAIT_OBJECT_0:
- code = _winapi.GetExitCodeProcess(self._handle)
- if code == TERMINATE:
- code = -signal.SIGTERM
- self.returncode = code
-
- return self.returncode
-
- def poll(self):
- return self.wait(timeout=0)
-
- def terminate(self):
- if self.returncode is None:
- try:
- _winapi.TerminateProcess(int(self._handle), TERMINATE)
- except OSError:
- if self.wait(timeout=1.0) is None:
- raise
-
- kill = terminate
-
- def close(self):
- self.finalizer()
+
+ # send information to child
+ set_spawning_popen(self)
+ try:
+ reduction.dump(prep_data, to_child)
+ reduction.dump(process_obj, to_child)
+ finally:
+ set_spawning_popen(None)
+
+ def duplicate_for_child(self, handle):
+ assert self is get_spawning_popen()
+ return reduction.duplicate(handle, self.sentinel)
+
+ def wait(self, timeout=None):
+ if self.returncode is None:
+ if timeout is None:
+ msecs = _winapi.INFINITE
+ else:
+ msecs = max(0, int(timeout * 1000 + 0.5))
+
+ res = _winapi.WaitForSingleObject(int(self._handle), msecs)
+ if res == _winapi.WAIT_OBJECT_0:
+ code = _winapi.GetExitCodeProcess(self._handle)
+ if code == TERMINATE:
+ code = -signal.SIGTERM
+ self.returncode = code
+
+ return self.returncode
+
+ def poll(self):
+ return self.wait(timeout=0)
+
+ def terminate(self):
+ if self.returncode is None:
+ try:
+ _winapi.TerminateProcess(int(self._handle), TERMINATE)
+ except OSError:
+ if self.wait(timeout=1.0) is None:
+ raise
+
+ kill = terminate
+
+ def close(self):
+ self.finalizer()
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/process.py b/contrib/tools/python3/src/Lib/multiprocessing/process.py
index 0b2e0b45b23..af6e2bec4f7 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/process.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/process.py
@@ -1,52 +1,52 @@
-#
-# Module providing the `Process` class which emulates `threading.Thread`
-#
-# multiprocessing/process.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
+#
+# Module providing the `Process` class which emulates `threading.Thread`
+#
+# multiprocessing/process.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
__all__ = ['BaseProcess', 'current_process', 'active_children',
'parent_process']
-
-#
-# Imports
-#
-
-import os
-import sys
-import signal
-import itertools
-import threading
-from _weakrefset import WeakSet
-
-#
-#
-#
-
-try:
- ORIGINAL_DIR = os.path.abspath(os.getcwd())
-except OSError:
- ORIGINAL_DIR = None
-
-#
-# Public functions
-#
-
-def current_process():
- '''
- Return process object representing the current process
- '''
- return _current_process
-
-def active_children():
- '''
- Return list of process objects corresponding to live child processes
- '''
- _cleanup()
- return list(_children)
-
+
+#
+# Imports
+#
+
+import os
+import sys
+import signal
+import itertools
+import threading
+from _weakrefset import WeakSet
+
+#
+#
+#
+
+try:
+ ORIGINAL_DIR = os.path.abspath(os.getcwd())
+except OSError:
+ ORIGINAL_DIR = None
+
+#
+# Public functions
+#
+
+def current_process():
+ '''
+ Return process object representing the current process
+ '''
+ return _current_process
+
+def active_children():
+ '''
+ Return list of process objects corresponding to live child processes
+ '''
+ _cleanup()
+ return list(_children)
+
def parent_process():
'''
@@ -54,225 +54,225 @@ def parent_process():
'''
return _parent_process
-#
-#
-#
-
-def _cleanup():
- # check for processes which have finished
- for p in list(_children):
- if p._popen.poll() is not None:
- _children.discard(p)
-
-#
-# The `Process` class
-#
-
-class BaseProcess(object):
- '''
- Process objects represent activity that is run in a separate process
-
- The class is analogous to `threading.Thread`
- '''
- def _Popen(self):
- raise NotImplementedError
-
- def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
- *, daemon=None):
- assert group is None, 'group argument must be None for now'
- count = next(_process_counter)
- self._identity = _current_process._identity + (count,)
- self._config = _current_process._config.copy()
- self._parent_pid = os.getpid()
+#
+#
+#
+
+def _cleanup():
+ # check for processes which have finished
+ for p in list(_children):
+ if p._popen.poll() is not None:
+ _children.discard(p)
+
+#
+# The `Process` class
+#
+
+class BaseProcess(object):
+ '''
+ Process objects represent activity that is run in a separate process
+
+ The class is analogous to `threading.Thread`
+ '''
+ def _Popen(self):
+ raise NotImplementedError
+
+ def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
+ *, daemon=None):
+ assert group is None, 'group argument must be None for now'
+ count = next(_process_counter)
+ self._identity = _current_process._identity + (count,)
+ self._config = _current_process._config.copy()
+ self._parent_pid = os.getpid()
self._parent_name = _current_process.name
- self._popen = None
- self._closed = False
- self._target = target
- self._args = tuple(args)
- self._kwargs = dict(kwargs)
- self._name = name or type(self).__name__ + '-' + \
- ':'.join(str(i) for i in self._identity)
- if daemon is not None:
- self.daemon = daemon
- _dangling.add(self)
-
- def _check_closed(self):
- if self._closed:
- raise ValueError("process object is closed")
-
- def run(self):
- '''
- Method to be run in sub-process; can be overridden in sub-class
- '''
- if self._target:
- self._target(*self._args, **self._kwargs)
-
- def start(self):
- '''
- Start child process
- '''
- self._check_closed()
- assert self._popen is None, 'cannot start a process twice'
- assert self._parent_pid == os.getpid(), \
- 'can only start a process object created by current process'
- assert not _current_process._config.get('daemon'), \
- 'daemonic processes are not allowed to have children'
- _cleanup()
- self._popen = self._Popen(self)
- self._sentinel = self._popen.sentinel
- # Avoid a refcycle if the target function holds an indirect
- # reference to the process object (see bpo-30775)
- del self._target, self._args, self._kwargs
- _children.add(self)
-
- def terminate(self):
- '''
- Terminate process; sends SIGTERM signal or uses TerminateProcess()
- '''
- self._check_closed()
- self._popen.terminate()
-
- def kill(self):
- '''
- Terminate process; sends SIGKILL signal or uses TerminateProcess()
- '''
- self._check_closed()
- self._popen.kill()
-
- def join(self, timeout=None):
- '''
- Wait until child process terminates
- '''
- self._check_closed()
- assert self._parent_pid == os.getpid(), 'can only join a child process'
- assert self._popen is not None, 'can only join a started process'
- res = self._popen.wait(timeout)
- if res is not None:
- _children.discard(self)
-
- def is_alive(self):
- '''
- Return whether process is alive
- '''
- self._check_closed()
- if self is _current_process:
- return True
- assert self._parent_pid == os.getpid(), 'can only test a child process'
-
- if self._popen is None:
- return False
-
- returncode = self._popen.poll()
- if returncode is None:
- return True
- else:
- _children.discard(self)
- return False
-
- def close(self):
- '''
- Close the Process object.
-
- This method releases resources held by the Process object. It is
- an error to call this method if the child process is still running.
- '''
- if self._popen is not None:
- if self._popen.poll() is None:
- raise ValueError("Cannot close a process while it is still running. "
- "You should first call join() or terminate().")
- self._popen.close()
- self._popen = None
- del self._sentinel
- _children.discard(self)
- self._closed = True
-
- @property
- def name(self):
- return self._name
-
- @name.setter
- def name(self, name):
- assert isinstance(name, str), 'name must be a string'
- self._name = name
-
- @property
- def daemon(self):
- '''
- Return whether process is a daemon
- '''
- return self._config.get('daemon', False)
-
- @daemon.setter
- def daemon(self, daemonic):
- '''
- Set whether process is a daemon
- '''
- assert self._popen is None, 'process has already started'
- self._config['daemon'] = daemonic
-
- @property
- def authkey(self):
- return self._config['authkey']
-
- @authkey.setter
- def authkey(self, authkey):
- '''
- Set authorization key of process
- '''
- self._config['authkey'] = AuthenticationString(authkey)
-
- @property
- def exitcode(self):
- '''
- Return exit code of process or `None` if it has yet to stop
- '''
- self._check_closed()
- if self._popen is None:
- return self._popen
- return self._popen.poll()
-
- @property
- def ident(self):
- '''
- Return identifier (PID) of process or `None` if it has yet to start
- '''
- self._check_closed()
- if self is _current_process:
- return os.getpid()
- else:
- return self._popen and self._popen.pid
-
- pid = ident
-
- @property
- def sentinel(self):
- '''
- Return a file descriptor (Unix) or handle (Windows) suitable for
- waiting for process termination.
- '''
- self._check_closed()
- try:
- return self._sentinel
- except AttributeError:
- raise ValueError("process not started") from None
-
- def __repr__(self):
+ self._popen = None
+ self._closed = False
+ self._target = target
+ self._args = tuple(args)
+ self._kwargs = dict(kwargs)
+ self._name = name or type(self).__name__ + '-' + \
+ ':'.join(str(i) for i in self._identity)
+ if daemon is not None:
+ self.daemon = daemon
+ _dangling.add(self)
+
+ def _check_closed(self):
+ if self._closed:
+ raise ValueError("process object is closed")
+
+ def run(self):
+ '''
+ Method to be run in sub-process; can be overridden in sub-class
+ '''
+ if self._target:
+ self._target(*self._args, **self._kwargs)
+
+ def start(self):
+ '''
+ Start child process
+ '''
+ self._check_closed()
+ assert self._popen is None, 'cannot start a process twice'
+ assert self._parent_pid == os.getpid(), \
+ 'can only start a process object created by current process'
+ assert not _current_process._config.get('daemon'), \
+ 'daemonic processes are not allowed to have children'
+ _cleanup()
+ self._popen = self._Popen(self)
+ self._sentinel = self._popen.sentinel
+ # Avoid a refcycle if the target function holds an indirect
+ # reference to the process object (see bpo-30775)
+ del self._target, self._args, self._kwargs
+ _children.add(self)
+
+ def terminate(self):
+ '''
+ Terminate process; sends SIGTERM signal or uses TerminateProcess()
+ '''
+ self._check_closed()
+ self._popen.terminate()
+
+ def kill(self):
+ '''
+ Terminate process; sends SIGKILL signal or uses TerminateProcess()
+ '''
+ self._check_closed()
+ self._popen.kill()
+
+ def join(self, timeout=None):
+ '''
+ Wait until child process terminates
+ '''
+ self._check_closed()
+ assert self._parent_pid == os.getpid(), 'can only join a child process'
+ assert self._popen is not None, 'can only join a started process'
+ res = self._popen.wait(timeout)
+ if res is not None:
+ _children.discard(self)
+
+ def is_alive(self):
+ '''
+ Return whether process is alive
+ '''
+ self._check_closed()
+ if self is _current_process:
+ return True
+ assert self._parent_pid == os.getpid(), 'can only test a child process'
+
+ if self._popen is None:
+ return False
+
+ returncode = self._popen.poll()
+ if returncode is None:
+ return True
+ else:
+ _children.discard(self)
+ return False
+
+ def close(self):
+ '''
+ Close the Process object.
+
+ This method releases resources held by the Process object. It is
+ an error to call this method if the child process is still running.
+ '''
+ if self._popen is not None:
+ if self._popen.poll() is None:
+ raise ValueError("Cannot close a process while it is still running. "
+ "You should first call join() or terminate().")
+ self._popen.close()
+ self._popen = None
+ del self._sentinel
+ _children.discard(self)
+ self._closed = True
+
+ @property
+ def name(self):
+ return self._name
+
+ @name.setter
+ def name(self, name):
+ assert isinstance(name, str), 'name must be a string'
+ self._name = name
+
+ @property
+ def daemon(self):
+ '''
+ Return whether process is a daemon
+ '''
+ return self._config.get('daemon', False)
+
+ @daemon.setter
+ def daemon(self, daemonic):
+ '''
+ Set whether process is a daemon
+ '''
+ assert self._popen is None, 'process has already started'
+ self._config['daemon'] = daemonic
+
+ @property
+ def authkey(self):
+ return self._config['authkey']
+
+ @authkey.setter
+ def authkey(self, authkey):
+ '''
+ Set authorization key of process
+ '''
+ self._config['authkey'] = AuthenticationString(authkey)
+
+ @property
+ def exitcode(self):
+ '''
+ Return exit code of process or `None` if it has yet to stop
+ '''
+ self._check_closed()
+ if self._popen is None:
+ return self._popen
+ return self._popen.poll()
+
+ @property
+ def ident(self):
+ '''
+ Return identifier (PID) of process or `None` if it has yet to start
+ '''
+ self._check_closed()
+ if self is _current_process:
+ return os.getpid()
+ else:
+ return self._popen and self._popen.pid
+
+ pid = ident
+
+ @property
+ def sentinel(self):
+ '''
+ Return a file descriptor (Unix) or handle (Windows) suitable for
+ waiting for process termination.
+ '''
+ self._check_closed()
+ try:
+ return self._sentinel
+ except AttributeError:
+ raise ValueError("process not started") from None
+
+ def __repr__(self):
exitcode = None
- if self is _current_process:
- status = 'started'
- elif self._closed:
- status = 'closed'
- elif self._parent_pid != os.getpid():
- status = 'unknown'
- elif self._popen is None:
- status = 'initial'
- else:
+ if self is _current_process:
+ status = 'started'
+ elif self._closed:
+ status = 'closed'
+ elif self._parent_pid != os.getpid():
+ status = 'unknown'
+ elif self._popen is None:
+ status = 'initial'
+ else:
exitcode = self._popen.poll()
if exitcode is not None:
status = 'stopped'
- else:
- status = 'started'
-
+ else:
+ status = 'started'
+
info = [type(self).__name__, 'name=%r' % self._name]
if self._popen is not None:
info.append('pid=%s' % self._popen.pid)
@@ -284,74 +284,74 @@ class BaseProcess(object):
if self.daemon:
info.append('daemon')
return '<%s>' % ' '.join(info)
-
- ##
-
+
+ ##
+
def _bootstrap(self, parent_sentinel=None):
- from . import util, context
+ from . import util, context
global _current_process, _parent_process, _process_counter, _children
-
- try:
- if self._start_method is not None:
- context._force_start_method(self._start_method)
- _process_counter = itertools.count(1)
- _children = set()
- util._close_stdin()
- old_process = _current_process
- _current_process = self
+
+ try:
+ if self._start_method is not None:
+ context._force_start_method(self._start_method)
+ _process_counter = itertools.count(1)
+ _children = set()
+ util._close_stdin()
+ old_process = _current_process
+ _current_process = self
_parent_process = _ParentProcess(
self._parent_name, self._parent_pid, parent_sentinel)
if threading._HAVE_THREAD_NATIVE_ID:
threading.main_thread()._set_native_id()
- try:
- util._finalizer_registry.clear()
- util._run_after_forkers()
- finally:
- # delay finalization of the old process object until after
- # _run_after_forkers() is executed
- del old_process
- util.info('child process calling self.run()')
- try:
- self.run()
- exitcode = 0
- finally:
- util._exit_function()
- except SystemExit as e:
+ try:
+ util._finalizer_registry.clear()
+ util._run_after_forkers()
+ finally:
+ # delay finalization of the old process object until after
+ # _run_after_forkers() is executed
+ del old_process
+ util.info('child process calling self.run()')
+ try:
+ self.run()
+ exitcode = 0
+ finally:
+ util._exit_function()
+ except SystemExit as e:
if e.code is None:
exitcode = 0
elif isinstance(e.code, int):
exitcode = e.code
- else:
+ else:
sys.stderr.write(str(e.code) + '\n')
- exitcode = 1
- except:
- exitcode = 1
- import traceback
- sys.stderr.write('Process %s:\n' % self.name)
- traceback.print_exc()
- finally:
- threading._shutdown()
- util.info('process exiting with exitcode %d' % exitcode)
- util._flush_std_streams()
+ exitcode = 1
+ except:
+ exitcode = 1
+ import traceback
+ sys.stderr.write('Process %s:\n' % self.name)
+ traceback.print_exc()
+ finally:
+ threading._shutdown()
+ util.info('process exiting with exitcode %d' % exitcode)
+ util._flush_std_streams()
+
+ return exitcode
+
+#
+# We subclass bytes to avoid accidental transmission of auth keys over network
+#
+
+class AuthenticationString(bytes):
+ def __reduce__(self):
+ from .context import get_spawning_popen
+ if get_spawning_popen() is None:
+ raise TypeError(
+ 'Pickling an AuthenticationString object is '
+ 'disallowed for security reasons'
+ )
+ return AuthenticationString, (bytes(self),)
+
- return exitcode
-
-#
-# We subclass bytes to avoid accidental transmission of auth keys over network
-#
-
-class AuthenticationString(bytes):
- def __reduce__(self):
- from .context import get_spawning_popen
- if get_spawning_popen() is None:
- raise TypeError(
- 'Pickling an AuthenticationString object is '
- 'disallowed for security reasons'
- )
- return AuthenticationString, (bytes(self),)
-
-
-#
+#
# Create object representing the parent process
#
@@ -385,48 +385,48 @@ class _ParentProcess(BaseProcess):
pid = ident
#
-# Create object representing the main process
-#
-
-class _MainProcess(BaseProcess):
-
- def __init__(self):
- self._identity = ()
- self._name = 'MainProcess'
- self._parent_pid = None
- self._popen = None
- self._closed = False
- self._config = {'authkey': AuthenticationString(os.urandom(32)),
- 'semprefix': '/mp'}
- # Note that some versions of FreeBSD only allow named
- # semaphores to have names of up to 14 characters. Therefore
- # we choose a short prefix.
- #
- # On MacOSX in a sandbox it may be necessary to use a
- # different prefix -- see #19478.
- #
- # Everything in self._config will be inherited by descendant
- # processes.
-
- def close(self):
- pass
-
-
+# Create object representing the main process
+#
+
+class _MainProcess(BaseProcess):
+
+ def __init__(self):
+ self._identity = ()
+ self._name = 'MainProcess'
+ self._parent_pid = None
+ self._popen = None
+ self._closed = False
+ self._config = {'authkey': AuthenticationString(os.urandom(32)),
+ 'semprefix': '/mp'}
+ # Note that some versions of FreeBSD only allow named
+ # semaphores to have names of up to 14 characters. Therefore
+ # we choose a short prefix.
+ #
+ # On MacOSX in a sandbox it may be necessary to use a
+ # different prefix -- see #19478.
+ #
+ # Everything in self._config will be inherited by descendant
+ # processes.
+
+ def close(self):
+ pass
+
+
_parent_process = None
-_current_process = _MainProcess()
-_process_counter = itertools.count(1)
-_children = set()
-del _MainProcess
-
-#
-# Give names to some return codes
-#
-
-_exitcode_to_name = {}
-
-for name, signum in list(signal.__dict__.items()):
- if name[:3]=='SIG' and '_' not in name:
+_current_process = _MainProcess()
+_process_counter = itertools.count(1)
+_children = set()
+del _MainProcess
+
+#
+# Give names to some return codes
+#
+
+_exitcode_to_name = {}
+
+for name, signum in list(signal.__dict__.items()):
+ if name[:3]=='SIG' and '_' not in name:
_exitcode_to_name[-signum] = f'-{name}'
-
-# For debug and leak testing
-_dangling = WeakSet()
+
+# For debug and leak testing
+_dangling = WeakSet()
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/queues.py b/contrib/tools/python3/src/Lib/multiprocessing/queues.py
index a2901814876..b2da9cef602 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/queues.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/queues.py
@@ -1,71 +1,71 @@
-#
-# Module implementing queues
-#
-# multiprocessing/queues.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
-
-import sys
-import os
-import threading
-import collections
-import time
+#
+# Module implementing queues
+#
+# multiprocessing/queues.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
+
+import sys
+import os
+import threading
+import collections
+import time
import types
-import weakref
-import errno
-
-from queue import Empty, Full
-
-import _multiprocessing
-
-from . import connection
-from . import context
-_ForkingPickler = context.reduction.ForkingPickler
-
-from .util import debug, info, Finalize, register_after_fork, is_exiting
-
-#
-# Queue type using a pipe, buffer and thread
-#
-
-class Queue(object):
-
- def __init__(self, maxsize=0, *, ctx):
- if maxsize <= 0:
- # Can raise ImportError (see issues #3770 and #23400)
- from .synchronize import SEM_VALUE_MAX as maxsize
- self._maxsize = maxsize
- self._reader, self._writer = connection.Pipe(duplex=False)
- self._rlock = ctx.Lock()
- self._opid = os.getpid()
- if sys.platform == 'win32':
- self._wlock = None
- else:
- self._wlock = ctx.Lock()
- self._sem = ctx.BoundedSemaphore(maxsize)
- # For use by concurrent.futures
- self._ignore_epipe = False
+import weakref
+import errno
+
+from queue import Empty, Full
+
+import _multiprocessing
+
+from . import connection
+from . import context
+_ForkingPickler = context.reduction.ForkingPickler
+
+from .util import debug, info, Finalize, register_after_fork, is_exiting
+
+#
+# Queue type using a pipe, buffer and thread
+#
+
+class Queue(object):
+
+ def __init__(self, maxsize=0, *, ctx):
+ if maxsize <= 0:
+ # Can raise ImportError (see issues #3770 and #23400)
+ from .synchronize import SEM_VALUE_MAX as maxsize
+ self._maxsize = maxsize
+ self._reader, self._writer = connection.Pipe(duplex=False)
+ self._rlock = ctx.Lock()
+ self._opid = os.getpid()
+ if sys.platform == 'win32':
+ self._wlock = None
+ else:
+ self._wlock = ctx.Lock()
+ self._sem = ctx.BoundedSemaphore(maxsize)
+ # For use by concurrent.futures
+ self._ignore_epipe = False
self._reset()
-
- if sys.platform != 'win32':
- register_after_fork(self, Queue._after_fork)
-
- def __getstate__(self):
- context.assert_spawning(self)
- return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
- self._rlock, self._wlock, self._sem, self._opid)
-
- def __setstate__(self, state):
- (self._ignore_epipe, self._maxsize, self._reader, self._writer,
- self._rlock, self._wlock, self._sem, self._opid) = state
+
+ if sys.platform != 'win32':
+ register_after_fork(self, Queue._after_fork)
+
+ def __getstate__(self):
+ context.assert_spawning(self)
+ return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
+ self._rlock, self._wlock, self._sem, self._opid)
+
+ def __setstate__(self, state):
+ (self._ignore_epipe, self._maxsize, self._reader, self._writer,
+ self._rlock, self._wlock, self._sem, self._opid) = state
self._reset()
-
- def _after_fork(self):
- debug('Queue._after_fork()')
+
+ def _after_fork(self):
+ debug('Queue._after_fork()')
self._reset(after_fork=True)
def _reset(self, after_fork=False):
@@ -73,308 +73,308 @@ class Queue(object):
self._notempty._at_fork_reinit()
else:
self._notempty = threading.Condition(threading.Lock())
- self._buffer = collections.deque()
- self._thread = None
- self._jointhread = None
- self._joincancelled = False
- self._closed = False
- self._close = None
- self._send_bytes = self._writer.send_bytes
- self._recv_bytes = self._reader.recv_bytes
- self._poll = self._reader.poll
-
- def put(self, obj, block=True, timeout=None):
+ self._buffer = collections.deque()
+ self._thread = None
+ self._jointhread = None
+ self._joincancelled = False
+ self._closed = False
+ self._close = None
+ self._send_bytes = self._writer.send_bytes
+ self._recv_bytes = self._reader.recv_bytes
+ self._poll = self._reader.poll
+
+ def put(self, obj, block=True, timeout=None):
if self._closed:
raise ValueError(f"Queue {self!r} is closed")
- if not self._sem.acquire(block, timeout):
- raise Full
-
- with self._notempty:
- if self._thread is None:
- self._start_thread()
- self._buffer.append(obj)
- self._notempty.notify()
-
- def get(self, block=True, timeout=None):
+ if not self._sem.acquire(block, timeout):
+ raise Full
+
+ with self._notempty:
+ if self._thread is None:
+ self._start_thread()
+ self._buffer.append(obj)
+ self._notempty.notify()
+
+ def get(self, block=True, timeout=None):
if self._closed:
raise ValueError(f"Queue {self!r} is closed")
- if block and timeout is None:
- with self._rlock:
- res = self._recv_bytes()
- self._sem.release()
- else:
- if block:
- deadline = time.monotonic() + timeout
- if not self._rlock.acquire(block, timeout):
- raise Empty
- try:
- if block:
- timeout = deadline - time.monotonic()
- if not self._poll(timeout):
- raise Empty
- elif not self._poll():
- raise Empty
- res = self._recv_bytes()
- self._sem.release()
- finally:
- self._rlock.release()
- # unserialize the data after having released the lock
- return _ForkingPickler.loads(res)
-
- def qsize(self):
- # Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
- return self._maxsize - self._sem._semlock._get_value()
-
- def empty(self):
- return not self._poll()
-
- def full(self):
- return self._sem._semlock._is_zero()
-
- def get_nowait(self):
- return self.get(False)
-
- def put_nowait(self, obj):
- return self.put(obj, False)
-
- def close(self):
- self._closed = True
- try:
- self._reader.close()
- finally:
- close = self._close
- if close:
- self._close = None
- close()
-
- def join_thread(self):
- debug('Queue.join_thread()')
- assert self._closed, "Queue {0!r} not closed".format(self)
- if self._jointhread:
- self._jointhread()
-
- def cancel_join_thread(self):
- debug('Queue.cancel_join_thread()')
- self._joincancelled = True
- try:
- self._jointhread.cancel()
- except AttributeError:
- pass
-
- def _start_thread(self):
- debug('Queue._start_thread()')
-
- # Start thread which transfers data from buffer to pipe
- self._buffer.clear()
- self._thread = threading.Thread(
- target=Queue._feed,
- args=(self._buffer, self._notempty, self._send_bytes,
- self._wlock, self._writer.close, self._ignore_epipe,
- self._on_queue_feeder_error, self._sem),
- name='QueueFeederThread'
- )
- self._thread.daemon = True
-
- debug('doing self._thread.start()')
- self._thread.start()
- debug('... done self._thread.start()')
-
- if not self._joincancelled:
- self._jointhread = Finalize(
- self._thread, Queue._finalize_join,
- [weakref.ref(self._thread)],
- exitpriority=-5
- )
-
- # Send sentinel to the thread queue object when garbage collected
- self._close = Finalize(
- self, Queue._finalize_close,
- [self._buffer, self._notempty],
- exitpriority=10
- )
-
- @staticmethod
- def _finalize_join(twr):
- debug('joining queue thread')
- thread = twr()
- if thread is not None:
- thread.join()
- debug('... queue thread joined')
- else:
- debug('... queue thread already dead')
-
- @staticmethod
- def _finalize_close(buffer, notempty):
- debug('telling queue thread to quit')
- with notempty:
- buffer.append(_sentinel)
- notempty.notify()
-
- @staticmethod
- def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe,
- onerror, queue_sem):
- debug('starting thread to feed data to pipe')
- nacquire = notempty.acquire
- nrelease = notempty.release
- nwait = notempty.wait
- bpopleft = buffer.popleft
- sentinel = _sentinel
- if sys.platform != 'win32':
- wacquire = writelock.acquire
- wrelease = writelock.release
- else:
- wacquire = None
-
- while 1:
- try:
- nacquire()
- try:
- if not buffer:
- nwait()
- finally:
- nrelease()
- try:
- while 1:
- obj = bpopleft()
- if obj is sentinel:
- debug('feeder thread got sentinel -- exiting')
- close()
- return
-
- # serialize the data before acquiring the lock
- obj = _ForkingPickler.dumps(obj)
- if wacquire is None:
- send_bytes(obj)
- else:
- wacquire()
- try:
- send_bytes(obj)
- finally:
- wrelease()
- except IndexError:
- pass
- except Exception as e:
- if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
- return
- # Since this runs in a daemon thread the resources it uses
- # may be become unusable while the process is cleaning up.
- # We ignore errors which happen after the process has
- # started to cleanup.
- if is_exiting():
- info('error in queue thread: %s', e)
- return
- else:
- # Since the object has not been sent in the queue, we need
- # to decrease the size of the queue. The error acts as
- # if the object had been silently removed from the queue
- # and this step is necessary to have a properly working
- # queue.
- queue_sem.release()
- onerror(e, obj)
-
- @staticmethod
- def _on_queue_feeder_error(e, obj):
- """
- Private API hook called when feeding data in the background thread
- raises an exception. For overriding by concurrent.futures.
- """
- import traceback
- traceback.print_exc()
-
-
-_sentinel = object()
-
-#
-# A queue type which also supports join() and task_done() methods
-#
-# Note that if you do not call task_done() for each finished task then
-# eventually the counter's semaphore may overflow causing Bad Things
-# to happen.
-#
-
-class JoinableQueue(Queue):
-
- def __init__(self, maxsize=0, *, ctx):
- Queue.__init__(self, maxsize, ctx=ctx)
- self._unfinished_tasks = ctx.Semaphore(0)
- self._cond = ctx.Condition()
-
- def __getstate__(self):
- return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
-
- def __setstate__(self, state):
- Queue.__setstate__(self, state[:-2])
- self._cond, self._unfinished_tasks = state[-2:]
-
- def put(self, obj, block=True, timeout=None):
+ if block and timeout is None:
+ with self._rlock:
+ res = self._recv_bytes()
+ self._sem.release()
+ else:
+ if block:
+ deadline = time.monotonic() + timeout
+ if not self._rlock.acquire(block, timeout):
+ raise Empty
+ try:
+ if block:
+ timeout = deadline - time.monotonic()
+ if not self._poll(timeout):
+ raise Empty
+ elif not self._poll():
+ raise Empty
+ res = self._recv_bytes()
+ self._sem.release()
+ finally:
+ self._rlock.release()
+ # unserialize the data after having released the lock
+ return _ForkingPickler.loads(res)
+
+ def qsize(self):
+ # Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
+ return self._maxsize - self._sem._semlock._get_value()
+
+ def empty(self):
+ return not self._poll()
+
+ def full(self):
+ return self._sem._semlock._is_zero()
+
+ def get_nowait(self):
+ return self.get(False)
+
+ def put_nowait(self, obj):
+ return self.put(obj, False)
+
+ def close(self):
+ self._closed = True
+ try:
+ self._reader.close()
+ finally:
+ close = self._close
+ if close:
+ self._close = None
+ close()
+
+ def join_thread(self):
+ debug('Queue.join_thread()')
+ assert self._closed, "Queue {0!r} not closed".format(self)
+ if self._jointhread:
+ self._jointhread()
+
+ def cancel_join_thread(self):
+ debug('Queue.cancel_join_thread()')
+ self._joincancelled = True
+ try:
+ self._jointhread.cancel()
+ except AttributeError:
+ pass
+
+ def _start_thread(self):
+ debug('Queue._start_thread()')
+
+ # Start thread which transfers data from buffer to pipe
+ self._buffer.clear()
+ self._thread = threading.Thread(
+ target=Queue._feed,
+ args=(self._buffer, self._notempty, self._send_bytes,
+ self._wlock, self._writer.close, self._ignore_epipe,
+ self._on_queue_feeder_error, self._sem),
+ name='QueueFeederThread'
+ )
+ self._thread.daemon = True
+
+ debug('doing self._thread.start()')
+ self._thread.start()
+ debug('... done self._thread.start()')
+
+ if not self._joincancelled:
+ self._jointhread = Finalize(
+ self._thread, Queue._finalize_join,
+ [weakref.ref(self._thread)],
+ exitpriority=-5
+ )
+
+ # Send sentinel to the thread queue object when garbage collected
+ self._close = Finalize(
+ self, Queue._finalize_close,
+ [self._buffer, self._notempty],
+ exitpriority=10
+ )
+
+ @staticmethod
+ def _finalize_join(twr):
+ debug('joining queue thread')
+ thread = twr()
+ if thread is not None:
+ thread.join()
+ debug('... queue thread joined')
+ else:
+ debug('... queue thread already dead')
+
+ @staticmethod
+ def _finalize_close(buffer, notempty):
+ debug('telling queue thread to quit')
+ with notempty:
+ buffer.append(_sentinel)
+ notempty.notify()
+
+ @staticmethod
+ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe,
+ onerror, queue_sem):
+ debug('starting thread to feed data to pipe')
+ nacquire = notempty.acquire
+ nrelease = notempty.release
+ nwait = notempty.wait
+ bpopleft = buffer.popleft
+ sentinel = _sentinel
+ if sys.platform != 'win32':
+ wacquire = writelock.acquire
+ wrelease = writelock.release
+ else:
+ wacquire = None
+
+ while 1:
+ try:
+ nacquire()
+ try:
+ if not buffer:
+ nwait()
+ finally:
+ nrelease()
+ try:
+ while 1:
+ obj = bpopleft()
+ if obj is sentinel:
+ debug('feeder thread got sentinel -- exiting')
+ close()
+ return
+
+ # serialize the data before acquiring the lock
+ obj = _ForkingPickler.dumps(obj)
+ if wacquire is None:
+ send_bytes(obj)
+ else:
+ wacquire()
+ try:
+ send_bytes(obj)
+ finally:
+ wrelease()
+ except IndexError:
+ pass
+ except Exception as e:
+ if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
+ return
+ # Since this runs in a daemon thread the resources it uses
+ # may be become unusable while the process is cleaning up.
+ # We ignore errors which happen after the process has
+ # started to cleanup.
+ if is_exiting():
+ info('error in queue thread: %s', e)
+ return
+ else:
+ # Since the object has not been sent in the queue, we need
+ # to decrease the size of the queue. The error acts as
+ # if the object had been silently removed from the queue
+ # and this step is necessary to have a properly working
+ # queue.
+ queue_sem.release()
+ onerror(e, obj)
+
+ @staticmethod
+ def _on_queue_feeder_error(e, obj):
+ """
+ Private API hook called when feeding data in the background thread
+ raises an exception. For overriding by concurrent.futures.
+ """
+ import traceback
+ traceback.print_exc()
+
+
+_sentinel = object()
+
+#
+# A queue type which also supports join() and task_done() methods
+#
+# Note that if you do not call task_done() for each finished task then
+# eventually the counter's semaphore may overflow causing Bad Things
+# to happen.
+#
+
+class JoinableQueue(Queue):
+
+ def __init__(self, maxsize=0, *, ctx):
+ Queue.__init__(self, maxsize, ctx=ctx)
+ self._unfinished_tasks = ctx.Semaphore(0)
+ self._cond = ctx.Condition()
+
+ def __getstate__(self):
+ return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
+
+ def __setstate__(self, state):
+ Queue.__setstate__(self, state[:-2])
+ self._cond, self._unfinished_tasks = state[-2:]
+
+ def put(self, obj, block=True, timeout=None):
if self._closed:
raise ValueError(f"Queue {self!r} is closed")
- if not self._sem.acquire(block, timeout):
- raise Full
-
- with self._notempty, self._cond:
- if self._thread is None:
- self._start_thread()
- self._buffer.append(obj)
- self._unfinished_tasks.release()
- self._notempty.notify()
-
- def task_done(self):
- with self._cond:
- if not self._unfinished_tasks.acquire(False):
- raise ValueError('task_done() called too many times')
- if self._unfinished_tasks._semlock._is_zero():
- self._cond.notify_all()
-
- def join(self):
- with self._cond:
- if not self._unfinished_tasks._semlock._is_zero():
- self._cond.wait()
-
-#
-# Simplified Queue type -- really just a locked pipe
-#
-
-class SimpleQueue(object):
-
- def __init__(self, *, ctx):
- self._reader, self._writer = connection.Pipe(duplex=False)
- self._rlock = ctx.Lock()
- self._poll = self._reader.poll
- if sys.platform == 'win32':
- self._wlock = None
- else:
- self._wlock = ctx.Lock()
-
+ if not self._sem.acquire(block, timeout):
+ raise Full
+
+ with self._notempty, self._cond:
+ if self._thread is None:
+ self._start_thread()
+ self._buffer.append(obj)
+ self._unfinished_tasks.release()
+ self._notempty.notify()
+
+ def task_done(self):
+ with self._cond:
+ if not self._unfinished_tasks.acquire(False):
+ raise ValueError('task_done() called too many times')
+ if self._unfinished_tasks._semlock._is_zero():
+ self._cond.notify_all()
+
+ def join(self):
+ with self._cond:
+ if not self._unfinished_tasks._semlock._is_zero():
+ self._cond.wait()
+
+#
+# Simplified Queue type -- really just a locked pipe
+#
+
+class SimpleQueue(object):
+
+ def __init__(self, *, ctx):
+ self._reader, self._writer = connection.Pipe(duplex=False)
+ self._rlock = ctx.Lock()
+ self._poll = self._reader.poll
+ if sys.platform == 'win32':
+ self._wlock = None
+ else:
+ self._wlock = ctx.Lock()
+
def close(self):
self._reader.close()
self._writer.close()
- def empty(self):
- return not self._poll()
-
- def __getstate__(self):
- context.assert_spawning(self)
- return (self._reader, self._writer, self._rlock, self._wlock)
-
- def __setstate__(self, state):
- (self._reader, self._writer, self._rlock, self._wlock) = state
- self._poll = self._reader.poll
-
- def get(self):
- with self._rlock:
- res = self._reader.recv_bytes()
- # unserialize the data after having released the lock
- return _ForkingPickler.loads(res)
-
- def put(self, obj):
- # serialize the data before acquiring the lock
- obj = _ForkingPickler.dumps(obj)
- if self._wlock is None:
- # writes to a message oriented win32 pipe are atomic
- self._writer.send_bytes(obj)
- else:
- with self._wlock:
- self._writer.send_bytes(obj)
+ def empty(self):
+ return not self._poll()
+
+ def __getstate__(self):
+ context.assert_spawning(self)
+ return (self._reader, self._writer, self._rlock, self._wlock)
+
+ def __setstate__(self, state):
+ (self._reader, self._writer, self._rlock, self._wlock) = state
+ self._poll = self._reader.poll
+
+ def get(self):
+ with self._rlock:
+ res = self._reader.recv_bytes()
+ # unserialize the data after having released the lock
+ return _ForkingPickler.loads(res)
+
+ def put(self, obj):
+ # serialize the data before acquiring the lock
+ obj = _ForkingPickler.dumps(obj)
+ if self._wlock is None:
+ # writes to a message oriented win32 pipe are atomic
+ self._writer.send_bytes(obj)
+ else:
+ with self._wlock:
+ self._writer.send_bytes(obj)
__class_getitem__ = classmethod(types.GenericAlias)
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/reduction.py b/contrib/tools/python3/src/Lib/multiprocessing/reduction.py
index 5593f0682f7..3c4df1dab4b 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/reduction.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/reduction.py
@@ -1,281 +1,281 @@
-#
-# Module which deals with pickling of objects.
-#
-# multiprocessing/reduction.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-from abc import ABCMeta
-import copyreg
-import functools
-import io
-import os
-import pickle
-import socket
-import sys
-
-from . import context
-
-__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump']
-
-
-HAVE_SEND_HANDLE = (sys.platform == 'win32' or
- (hasattr(socket, 'CMSG_LEN') and
- hasattr(socket, 'SCM_RIGHTS') and
- hasattr(socket.socket, 'sendmsg')))
-
-#
-# Pickler subclass
-#
-
-class ForkingPickler(pickle.Pickler):
- '''Pickler subclass used by multiprocessing.'''
- _extra_reducers = {}
- _copyreg_dispatch_table = copyreg.dispatch_table
-
- def __init__(self, *args):
- super().__init__(*args)
- self.dispatch_table = self._copyreg_dispatch_table.copy()
- self.dispatch_table.update(self._extra_reducers)
-
- @classmethod
- def register(cls, type, reduce):
- '''Register a reduce function for a type.'''
- cls._extra_reducers[type] = reduce
-
- @classmethod
- def dumps(cls, obj, protocol=None):
- buf = io.BytesIO()
- cls(buf, protocol).dump(obj)
- return buf.getbuffer()
-
- loads = pickle.loads
-
-register = ForkingPickler.register
-
-def dump(obj, file, protocol=None):
- '''Replacement for pickle.dump() using ForkingPickler.'''
- ForkingPickler(file, protocol).dump(obj)
-
-#
-# Platform specific definitions
-#
-
-if sys.platform == 'win32':
- # Windows
- __all__ += ['DupHandle', 'duplicate', 'steal_handle']
- import _winapi
-
+#
+# Module which deals with pickling of objects.
+#
+# multiprocessing/reduction.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+from abc import ABCMeta
+import copyreg
+import functools
+import io
+import os
+import pickle
+import socket
+import sys
+
+from . import context
+
+__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump']
+
+
+HAVE_SEND_HANDLE = (sys.platform == 'win32' or
+ (hasattr(socket, 'CMSG_LEN') and
+ hasattr(socket, 'SCM_RIGHTS') and
+ hasattr(socket.socket, 'sendmsg')))
+
+#
+# Pickler subclass
+#
+
+class ForkingPickler(pickle.Pickler):
+ '''Pickler subclass used by multiprocessing.'''
+ _extra_reducers = {}
+ _copyreg_dispatch_table = copyreg.dispatch_table
+
+ def __init__(self, *args):
+ super().__init__(*args)
+ self.dispatch_table = self._copyreg_dispatch_table.copy()
+ self.dispatch_table.update(self._extra_reducers)
+
+ @classmethod
+ def register(cls, type, reduce):
+ '''Register a reduce function for a type.'''
+ cls._extra_reducers[type] = reduce
+
+ @classmethod
+ def dumps(cls, obj, protocol=None):
+ buf = io.BytesIO()
+ cls(buf, protocol).dump(obj)
+ return buf.getbuffer()
+
+ loads = pickle.loads
+
+register = ForkingPickler.register
+
+def dump(obj, file, protocol=None):
+ '''Replacement for pickle.dump() using ForkingPickler.'''
+ ForkingPickler(file, protocol).dump(obj)
+
+#
+# Platform specific definitions
+#
+
+if sys.platform == 'win32':
+ # Windows
+ __all__ += ['DupHandle', 'duplicate', 'steal_handle']
+ import _winapi
+
def duplicate(handle, target_process=None, inheritable=False,
*, source_process=None):
- '''Duplicate a handle. (target_process is a handle not a pid!)'''
+ '''Duplicate a handle. (target_process is a handle not a pid!)'''
current_process = _winapi.GetCurrentProcess()
if source_process is None:
source_process = current_process
- if target_process is None:
+ if target_process is None:
target_process = current_process
- return _winapi.DuplicateHandle(
+ return _winapi.DuplicateHandle(
source_process, handle, target_process,
- 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
-
- def steal_handle(source_pid, handle):
- '''Steal a handle from process identified by source_pid.'''
- source_process_handle = _winapi.OpenProcess(
- _winapi.PROCESS_DUP_HANDLE, False, source_pid)
- try:
- return _winapi.DuplicateHandle(
- source_process_handle, handle,
- _winapi.GetCurrentProcess(), 0, False,
- _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE)
- finally:
- _winapi.CloseHandle(source_process_handle)
-
- def send_handle(conn, handle, destination_pid):
- '''Send a handle over a local connection.'''
- dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
- conn.send(dh)
-
- def recv_handle(conn):
- '''Receive a handle over a local connection.'''
- return conn.recv().detach()
-
- class DupHandle(object):
- '''Picklable wrapper for a handle.'''
- def __init__(self, handle, access, pid=None):
- if pid is None:
- # We just duplicate the handle in the current process and
- # let the receiving process steal the handle.
- pid = os.getpid()
- proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
- try:
- self._handle = _winapi.DuplicateHandle(
- _winapi.GetCurrentProcess(),
- handle, proc, access, False, 0)
- finally:
- _winapi.CloseHandle(proc)
- self._access = access
- self._pid = pid
-
- def detach(self):
- '''Get the handle. This should only be called once.'''
- # retrieve handle from process which currently owns it
- if self._pid == os.getpid():
- # The handle has already been duplicated for this process.
- return self._handle
- # We must steal the handle from the process whose pid is self._pid.
- proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
- self._pid)
- try:
- return _winapi.DuplicateHandle(
- proc, self._handle, _winapi.GetCurrentProcess(),
- self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE)
- finally:
- _winapi.CloseHandle(proc)
-
-else:
- # Unix
- __all__ += ['DupFd', 'sendfds', 'recvfds']
- import array
-
- # On MacOSX we should acknowledge receipt of fds -- see Issue14669
- ACKNOWLEDGE = sys.platform == 'darwin'
-
- def sendfds(sock, fds):
- '''Send an array of fds over an AF_UNIX socket.'''
- fds = array.array('i', fds)
- msg = bytes([len(fds) % 256])
- sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
- if ACKNOWLEDGE and sock.recv(1) != b'A':
- raise RuntimeError('did not receive acknowledgement of fd')
-
- def recvfds(sock, size):
- '''Receive an array of fds over an AF_UNIX socket.'''
- a = array.array('i')
- bytes_size = a.itemsize * size
- msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size))
- if not msg and not ancdata:
- raise EOFError
- try:
- if ACKNOWLEDGE:
- sock.send(b'A')
- if len(ancdata) != 1:
- raise RuntimeError('received %d items of ancdata' %
- len(ancdata))
- cmsg_level, cmsg_type, cmsg_data = ancdata[0]
- if (cmsg_level == socket.SOL_SOCKET and
- cmsg_type == socket.SCM_RIGHTS):
- if len(cmsg_data) % a.itemsize != 0:
- raise ValueError
- a.frombytes(cmsg_data)
- if len(a) % 256 != msg[0]:
- raise AssertionError(
- "Len is {0:n} but msg[0] is {1!r}".format(
- len(a), msg[0]))
- return list(a)
- except (ValueError, IndexError):
- pass
- raise RuntimeError('Invalid data received')
-
- def send_handle(conn, handle, destination_pid):
- '''Send a handle over a local connection.'''
- with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
- sendfds(s, [handle])
-
- def recv_handle(conn):
- '''Receive a handle over a local connection.'''
- with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
- return recvfds(s, 1)[0]
-
- def DupFd(fd):
- '''Return a wrapper for an fd.'''
- popen_obj = context.get_spawning_popen()
- if popen_obj is not None:
- return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
- elif HAVE_SEND_HANDLE:
- from . import resource_sharer
- return resource_sharer.DupFd(fd)
- else:
- raise ValueError('SCM_RIGHTS appears not to be available')
-
-#
-# Try making some callable types picklable
-#
-
-def _reduce_method(m):
- if m.__self__ is None:
- return getattr, (m.__class__, m.__func__.__name__)
- else:
- return getattr, (m.__self__, m.__func__.__name__)
-class _C:
- def f(self):
- pass
-register(type(_C().f), _reduce_method)
-
-
-def _reduce_method_descriptor(m):
- return getattr, (m.__objclass__, m.__name__)
-register(type(list.append), _reduce_method_descriptor)
-register(type(int.__add__), _reduce_method_descriptor)
-
-
-def _reduce_partial(p):
- return _rebuild_partial, (p.func, p.args, p.keywords or {})
-def _rebuild_partial(func, args, keywords):
- return functools.partial(func, *args, **keywords)
-register(functools.partial, _reduce_partial)
-
-#
-# Make sockets picklable
-#
-
-if sys.platform == 'win32':
- def _reduce_socket(s):
- from .resource_sharer import DupSocket
- return _rebuild_socket, (DupSocket(s),)
- def _rebuild_socket(ds):
- return ds.detach()
- register(socket.socket, _reduce_socket)
-
-else:
- def _reduce_socket(s):
- df = DupFd(s.fileno())
- return _rebuild_socket, (df, s.family, s.type, s.proto)
- def _rebuild_socket(df, family, type, proto):
- fd = df.detach()
- return socket.socket(family, type, proto, fileno=fd)
- register(socket.socket, _reduce_socket)
-
-
-class AbstractReducer(metaclass=ABCMeta):
- '''Abstract base class for use in implementing a Reduction class
- suitable for use in replacing the standard reduction mechanism
- used in multiprocessing.'''
- ForkingPickler = ForkingPickler
- register = register
- dump = dump
- send_handle = send_handle
- recv_handle = recv_handle
-
- if sys.platform == 'win32':
- steal_handle = steal_handle
- duplicate = duplicate
- DupHandle = DupHandle
- else:
- sendfds = sendfds
- recvfds = recvfds
- DupFd = DupFd
-
- _reduce_method = _reduce_method
- _reduce_method_descriptor = _reduce_method_descriptor
- _rebuild_partial = _rebuild_partial
- _reduce_socket = _reduce_socket
- _rebuild_socket = _rebuild_socket
-
- def __init__(self, *args):
- register(type(_C().f), _reduce_method)
- register(type(list.append), _reduce_method_descriptor)
- register(type(int.__add__), _reduce_method_descriptor)
- register(functools.partial, _reduce_partial)
- register(socket.socket, _reduce_socket)
+ 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
+
+ def steal_handle(source_pid, handle):
+ '''Steal a handle from process identified by source_pid.'''
+ source_process_handle = _winapi.OpenProcess(
+ _winapi.PROCESS_DUP_HANDLE, False, source_pid)
+ try:
+ return _winapi.DuplicateHandle(
+ source_process_handle, handle,
+ _winapi.GetCurrentProcess(), 0, False,
+ _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE)
+ finally:
+ _winapi.CloseHandle(source_process_handle)
+
+ def send_handle(conn, handle, destination_pid):
+ '''Send a handle over a local connection.'''
+ dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
+ conn.send(dh)
+
+ def recv_handle(conn):
+ '''Receive a handle over a local connection.'''
+ return conn.recv().detach()
+
+ class DupHandle(object):
+ '''Picklable wrapper for a handle.'''
+ def __init__(self, handle, access, pid=None):
+ if pid is None:
+ # We just duplicate the handle in the current process and
+ # let the receiving process steal the handle.
+ pid = os.getpid()
+ proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
+ try:
+ self._handle = _winapi.DuplicateHandle(
+ _winapi.GetCurrentProcess(),
+ handle, proc, access, False, 0)
+ finally:
+ _winapi.CloseHandle(proc)
+ self._access = access
+ self._pid = pid
+
+ def detach(self):
+ '''Get the handle. This should only be called once.'''
+ # retrieve handle from process which currently owns it
+ if self._pid == os.getpid():
+ # The handle has already been duplicated for this process.
+ return self._handle
+ # We must steal the handle from the process whose pid is self._pid.
+ proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
+ self._pid)
+ try:
+ return _winapi.DuplicateHandle(
+ proc, self._handle, _winapi.GetCurrentProcess(),
+ self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE)
+ finally:
+ _winapi.CloseHandle(proc)
+
+else:
+ # Unix
+ __all__ += ['DupFd', 'sendfds', 'recvfds']
+ import array
+
+ # On MacOSX we should acknowledge receipt of fds -- see Issue14669
+ ACKNOWLEDGE = sys.platform == 'darwin'
+
+ def sendfds(sock, fds):
+ '''Send an array of fds over an AF_UNIX socket.'''
+ fds = array.array('i', fds)
+ msg = bytes([len(fds) % 256])
+ sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
+ if ACKNOWLEDGE and sock.recv(1) != b'A':
+ raise RuntimeError('did not receive acknowledgement of fd')
+
+ def recvfds(sock, size):
+ '''Receive an array of fds over an AF_UNIX socket.'''
+ a = array.array('i')
+ bytes_size = a.itemsize * size
+ msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size))
+ if not msg and not ancdata:
+ raise EOFError
+ try:
+ if ACKNOWLEDGE:
+ sock.send(b'A')
+ if len(ancdata) != 1:
+ raise RuntimeError('received %d items of ancdata' %
+ len(ancdata))
+ cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+ if (cmsg_level == socket.SOL_SOCKET and
+ cmsg_type == socket.SCM_RIGHTS):
+ if len(cmsg_data) % a.itemsize != 0:
+ raise ValueError
+ a.frombytes(cmsg_data)
+ if len(a) % 256 != msg[0]:
+ raise AssertionError(
+ "Len is {0:n} but msg[0] is {1!r}".format(
+ len(a), msg[0]))
+ return list(a)
+ except (ValueError, IndexError):
+ pass
+ raise RuntimeError('Invalid data received')
+
+ def send_handle(conn, handle, destination_pid):
+ '''Send a handle over a local connection.'''
+ with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
+ sendfds(s, [handle])
+
+ def recv_handle(conn):
+ '''Receive a handle over a local connection.'''
+ with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
+ return recvfds(s, 1)[0]
+
+ def DupFd(fd):
+ '''Return a wrapper for an fd.'''
+ popen_obj = context.get_spawning_popen()
+ if popen_obj is not None:
+ return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
+ elif HAVE_SEND_HANDLE:
+ from . import resource_sharer
+ return resource_sharer.DupFd(fd)
+ else:
+ raise ValueError('SCM_RIGHTS appears not to be available')
+
+#
+# Try making some callable types picklable
+#
+
+def _reduce_method(m):
+ if m.__self__ is None:
+ return getattr, (m.__class__, m.__func__.__name__)
+ else:
+ return getattr, (m.__self__, m.__func__.__name__)
+class _C:
+ def f(self):
+ pass
+register(type(_C().f), _reduce_method)
+
+
+def _reduce_method_descriptor(m):
+ return getattr, (m.__objclass__, m.__name__)
+register(type(list.append), _reduce_method_descriptor)
+register(type(int.__add__), _reduce_method_descriptor)
+
+
+def _reduce_partial(p):
+ return _rebuild_partial, (p.func, p.args, p.keywords or {})
+def _rebuild_partial(func, args, keywords):
+ return functools.partial(func, *args, **keywords)
+register(functools.partial, _reduce_partial)
+
+#
+# Make sockets picklable
+#
+
+if sys.platform == 'win32':
+ def _reduce_socket(s):
+ from .resource_sharer import DupSocket
+ return _rebuild_socket, (DupSocket(s),)
+ def _rebuild_socket(ds):
+ return ds.detach()
+ register(socket.socket, _reduce_socket)
+
+else:
+ def _reduce_socket(s):
+ df = DupFd(s.fileno())
+ return _rebuild_socket, (df, s.family, s.type, s.proto)
+ def _rebuild_socket(df, family, type, proto):
+ fd = df.detach()
+ return socket.socket(family, type, proto, fileno=fd)
+ register(socket.socket, _reduce_socket)
+
+
+class AbstractReducer(metaclass=ABCMeta):
+ '''Abstract base class for use in implementing a Reduction class
+ suitable for use in replacing the standard reduction mechanism
+ used in multiprocessing.'''
+ ForkingPickler = ForkingPickler
+ register = register
+ dump = dump
+ send_handle = send_handle
+ recv_handle = recv_handle
+
+ if sys.platform == 'win32':
+ steal_handle = steal_handle
+ duplicate = duplicate
+ DupHandle = DupHandle
+ else:
+ sendfds = sendfds
+ recvfds = recvfds
+ DupFd = DupFd
+
+ _reduce_method = _reduce_method
+ _reduce_method_descriptor = _reduce_method_descriptor
+ _rebuild_partial = _rebuild_partial
+ _reduce_socket = _reduce_socket
+ _rebuild_socket = _rebuild_socket
+
+ def __init__(self, *args):
+ register(type(_C().f), _reduce_method)
+ register(type(list.append), _reduce_method_descriptor)
+ register(type(int.__add__), _reduce_method_descriptor)
+ register(functools.partial, _reduce_partial)
+ register(socket.socket, _reduce_socket)
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/resource_sharer.py b/contrib/tools/python3/src/Lib/multiprocessing/resource_sharer.py
index 66076509a12..e3cb5da6f62 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/resource_sharer.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/resource_sharer.py
@@ -1,154 +1,154 @@
-#
-# We use a background thread for sharing fds on Unix, and for sharing sockets on
-# Windows.
-#
-# A client which wants to pickle a resource registers it with the resource
-# sharer and gets an identifier in return. The unpickling process will connect
-# to the resource sharer, sends the identifier and its pid, and then receives
-# the resource.
-#
-
-import os
-import signal
-import socket
-import sys
-import threading
-
-from . import process
-from .context import reduction
-from . import util
-
-__all__ = ['stop']
-
-
-if sys.platform == 'win32':
- __all__ += ['DupSocket']
-
- class DupSocket(object):
- '''Picklable wrapper for a socket.'''
- def __init__(self, sock):
- new_sock = sock.dup()
- def send(conn, pid):
- share = new_sock.share(pid)
- conn.send_bytes(share)
- self._id = _resource_sharer.register(send, new_sock.close)
-
- def detach(self):
- '''Get the socket. This should only be called once.'''
- with _resource_sharer.get_connection(self._id) as conn:
- share = conn.recv_bytes()
- return socket.fromshare(share)
-
-else:
- __all__ += ['DupFd']
-
- class DupFd(object):
- '''Wrapper for fd which can be used at any time.'''
- def __init__(self, fd):
- new_fd = os.dup(fd)
- def send(conn, pid):
- reduction.send_handle(conn, new_fd, pid)
- def close():
- os.close(new_fd)
- self._id = _resource_sharer.register(send, close)
-
- def detach(self):
- '''Get the fd. This should only be called once.'''
- with _resource_sharer.get_connection(self._id) as conn:
- return reduction.recv_handle(conn)
-
-
-class _ResourceSharer(object):
+#
+# We use a background thread for sharing fds on Unix, and for sharing sockets on
+# Windows.
+#
+# A client which wants to pickle a resource registers it with the resource
+# sharer and gets an identifier in return. The unpickling process will connect
+# to the resource sharer, sends the identifier and its pid, and then receives
+# the resource.
+#
+
+import os
+import signal
+import socket
+import sys
+import threading
+
+from . import process
+from .context import reduction
+from . import util
+
+__all__ = ['stop']
+
+
+if sys.platform == 'win32':
+ __all__ += ['DupSocket']
+
+ class DupSocket(object):
+ '''Picklable wrapper for a socket.'''
+ def __init__(self, sock):
+ new_sock = sock.dup()
+ def send(conn, pid):
+ share = new_sock.share(pid)
+ conn.send_bytes(share)
+ self._id = _resource_sharer.register(send, new_sock.close)
+
+ def detach(self):
+ '''Get the socket. This should only be called once.'''
+ with _resource_sharer.get_connection(self._id) as conn:
+ share = conn.recv_bytes()
+ return socket.fromshare(share)
+
+else:
+ __all__ += ['DupFd']
+
+ class DupFd(object):
+ '''Wrapper for fd which can be used at any time.'''
+ def __init__(self, fd):
+ new_fd = os.dup(fd)
+ def send(conn, pid):
+ reduction.send_handle(conn, new_fd, pid)
+ def close():
+ os.close(new_fd)
+ self._id = _resource_sharer.register(send, close)
+
+ def detach(self):
+ '''Get the fd. This should only be called once.'''
+ with _resource_sharer.get_connection(self._id) as conn:
+ return reduction.recv_handle(conn)
+
+
+class _ResourceSharer(object):
'''Manager for resources using background thread.'''
- def __init__(self):
- self._key = 0
- self._cache = {}
- self._lock = threading.Lock()
- self._listener = None
- self._address = None
- self._thread = None
- util.register_after_fork(self, _ResourceSharer._afterfork)
-
- def register(self, send, close):
- '''Register resource, returning an identifier.'''
- with self._lock:
- if self._address is None:
- self._start()
- self._key += 1
- self._cache[self._key] = (send, close)
- return (self._address, self._key)
-
- @staticmethod
- def get_connection(ident):
- '''Return connection from which to receive identified resource.'''
- from .connection import Client
- address, key = ident
- c = Client(address, authkey=process.current_process().authkey)
- c.send((key, os.getpid()))
- return c
-
- def stop(self, timeout=None):
- '''Stop the background thread and clear registered resources.'''
- from .connection import Client
- with self._lock:
- if self._address is not None:
- c = Client(self._address,
- authkey=process.current_process().authkey)
- c.send(None)
- c.close()
- self._thread.join(timeout)
- if self._thread.is_alive():
- util.sub_warning('_ResourceSharer thread did '
- 'not stop when asked')
- self._listener.close()
- self._thread = None
- self._address = None
- self._listener = None
- for key, (send, close) in self._cache.items():
- close()
- self._cache.clear()
-
- def _afterfork(self):
- for key, (send, close) in self._cache.items():
- close()
- self._cache.clear()
+ def __init__(self):
+ self._key = 0
+ self._cache = {}
+ self._lock = threading.Lock()
+ self._listener = None
+ self._address = None
+ self._thread = None
+ util.register_after_fork(self, _ResourceSharer._afterfork)
+
+ def register(self, send, close):
+ '''Register resource, returning an identifier.'''
+ with self._lock:
+ if self._address is None:
+ self._start()
+ self._key += 1
+ self._cache[self._key] = (send, close)
+ return (self._address, self._key)
+
+ @staticmethod
+ def get_connection(ident):
+ '''Return connection from which to receive identified resource.'''
+ from .connection import Client
+ address, key = ident
+ c = Client(address, authkey=process.current_process().authkey)
+ c.send((key, os.getpid()))
+ return c
+
+ def stop(self, timeout=None):
+ '''Stop the background thread and clear registered resources.'''
+ from .connection import Client
+ with self._lock:
+ if self._address is not None:
+ c = Client(self._address,
+ authkey=process.current_process().authkey)
+ c.send(None)
+ c.close()
+ self._thread.join(timeout)
+ if self._thread.is_alive():
+ util.sub_warning('_ResourceSharer thread did '
+ 'not stop when asked')
+ self._listener.close()
+ self._thread = None
+ self._address = None
+ self._listener = None
+ for key, (send, close) in self._cache.items():
+ close()
+ self._cache.clear()
+
+ def _afterfork(self):
+ for key, (send, close) in self._cache.items():
+ close()
+ self._cache.clear()
self._lock._at_fork_reinit()
- if self._listener is not None:
- self._listener.close()
- self._listener = None
- self._address = None
- self._thread = None
-
- def _start(self):
- from .connection import Listener
- assert self._listener is None, "Already have Listener"
- util.debug('starting listener and thread for sending handles')
- self._listener = Listener(authkey=process.current_process().authkey)
- self._address = self._listener.address
- t = threading.Thread(target=self._serve)
- t.daemon = True
- t.start()
- self._thread = t
-
- def _serve(self):
- if hasattr(signal, 'pthread_sigmask'):
+ if self._listener is not None:
+ self._listener.close()
+ self._listener = None
+ self._address = None
+ self._thread = None
+
+ def _start(self):
+ from .connection import Listener
+ assert self._listener is None, "Already have Listener"
+ util.debug('starting listener and thread for sending handles')
+ self._listener = Listener(authkey=process.current_process().authkey)
+ self._address = self._listener.address
+ t = threading.Thread(target=self._serve)
+ t.daemon = True
+ t.start()
+ self._thread = t
+
+ def _serve(self):
+ if hasattr(signal, 'pthread_sigmask'):
signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals())
- while 1:
- try:
- with self._listener.accept() as conn:
- msg = conn.recv()
- if msg is None:
- break
- key, destination_pid = msg
- send, close = self._cache.pop(key)
- try:
- send(conn, destination_pid)
- finally:
- close()
- except:
- if not util.is_exiting():
- sys.excepthook(*sys.exc_info())
-
-
-_resource_sharer = _ResourceSharer()
-stop = _resource_sharer.stop
+ while 1:
+ try:
+ with self._listener.accept() as conn:
+ msg = conn.recv()
+ if msg is None:
+ break
+ key, destination_pid = msg
+ send, close = self._cache.pop(key)
+ try:
+ send(conn, destination_pid)
+ finally:
+ close()
+ except:
+ if not util.is_exiting():
+ sys.excepthook(*sys.exc_info())
+
+
+_resource_sharer = _ResourceSharer()
+stop = _resource_sharer.stop
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/sharedctypes.py b/contrib/tools/python3/src/Lib/multiprocessing/sharedctypes.py
index 6071707027b..ed94d765829 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/sharedctypes.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/sharedctypes.py
@@ -1,240 +1,240 @@
-#
-# Module which supports allocation of ctypes objects from shared memory
-#
-# multiprocessing/sharedctypes.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-import ctypes
-import weakref
-
-from . import heap
-from . import get_context
-
-from .context import reduction, assert_spawning
-_ForkingPickler = reduction.ForkingPickler
-
-__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
-
-#
-#
-#
-
-typecode_to_type = {
- 'c': ctypes.c_char, 'u': ctypes.c_wchar,
- 'b': ctypes.c_byte, 'B': ctypes.c_ubyte,
- 'h': ctypes.c_short, 'H': ctypes.c_ushort,
- 'i': ctypes.c_int, 'I': ctypes.c_uint,
- 'l': ctypes.c_long, 'L': ctypes.c_ulong,
- 'q': ctypes.c_longlong, 'Q': ctypes.c_ulonglong,
- 'f': ctypes.c_float, 'd': ctypes.c_double
- }
-
-#
-#
-#
-
-def _new_value(type_):
- size = ctypes.sizeof(type_)
- wrapper = heap.BufferWrapper(size)
- return rebuild_ctype(type_, wrapper, None)
-
-def RawValue(typecode_or_type, *args):
- '''
- Returns a ctypes object allocated from shared memory
- '''
- type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
- obj = _new_value(type_)
- ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
- obj.__init__(*args)
- return obj
-
-def RawArray(typecode_or_type, size_or_initializer):
- '''
- Returns a ctypes array allocated from shared memory
- '''
- type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
- if isinstance(size_or_initializer, int):
- type_ = type_ * size_or_initializer
- obj = _new_value(type_)
- ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
- return obj
- else:
- type_ = type_ * len(size_or_initializer)
- result = _new_value(type_)
- result.__init__(*size_or_initializer)
- return result
-
-def Value(typecode_or_type, *args, lock=True, ctx=None):
- '''
- Return a synchronization wrapper for a Value
- '''
- obj = RawValue(typecode_or_type, *args)
- if lock is False:
- return obj
- if lock in (True, None):
- ctx = ctx or get_context()
- lock = ctx.RLock()
- if not hasattr(lock, 'acquire'):
- raise AttributeError("%r has no method 'acquire'" % lock)
- return synchronized(obj, lock, ctx=ctx)
-
-def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None):
- '''
- Return a synchronization wrapper for a RawArray
- '''
- obj = RawArray(typecode_or_type, size_or_initializer)
- if lock is False:
- return obj
- if lock in (True, None):
- ctx = ctx or get_context()
- lock = ctx.RLock()
- if not hasattr(lock, 'acquire'):
- raise AttributeError("%r has no method 'acquire'" % lock)
- return synchronized(obj, lock, ctx=ctx)
-
-def copy(obj):
- new_obj = _new_value(type(obj))
- ctypes.pointer(new_obj)[0] = obj
- return new_obj
-
-def synchronized(obj, lock=None, ctx=None):
- assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
- ctx = ctx or get_context()
-
- if isinstance(obj, ctypes._SimpleCData):
- return Synchronized(obj, lock, ctx)
- elif isinstance(obj, ctypes.Array):
- if obj._type_ is ctypes.c_char:
- return SynchronizedString(obj, lock, ctx)
- return SynchronizedArray(obj, lock, ctx)
- else:
- cls = type(obj)
- try:
- scls = class_cache[cls]
- except KeyError:
- names = [field[0] for field in cls._fields_]
- d = {name: make_property(name) for name in names}
- classname = 'Synchronized' + cls.__name__
- scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
- return scls(obj, lock, ctx)
-
-#
-# Functions for pickling/unpickling
-#
-
-def reduce_ctype(obj):
- assert_spawning(obj)
- if isinstance(obj, ctypes.Array):
- return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)
- else:
- return rebuild_ctype, (type(obj), obj._wrapper, None)
-
-def rebuild_ctype(type_, wrapper, length):
- if length is not None:
- type_ = type_ * length
- _ForkingPickler.register(type_, reduce_ctype)
- buf = wrapper.create_memoryview()
- obj = type_.from_buffer(buf)
- obj._wrapper = wrapper
- return obj
-
-#
-# Function to create properties
-#
-
-def make_property(name):
- try:
- return prop_cache[name]
- except KeyError:
- d = {}
- exec(template % ((name,)*7), d)
- prop_cache[name] = d[name]
- return d[name]
-
-template = '''
-def get%s(self):
- self.acquire()
- try:
- return self._obj.%s
- finally:
- self.release()
-def set%s(self, value):
- self.acquire()
- try:
- self._obj.%s = value
- finally:
- self.release()
-%s = property(get%s, set%s)
-'''
-
-prop_cache = {}
-class_cache = weakref.WeakKeyDictionary()
-
-#
-# Synchronized wrappers
-#
-
-class SynchronizedBase(object):
-
- def __init__(self, obj, lock=None, ctx=None):
- self._obj = obj
- if lock:
- self._lock = lock
- else:
- ctx = ctx or get_context(force=True)
- self._lock = ctx.RLock()
- self.acquire = self._lock.acquire
- self.release = self._lock.release
-
- def __enter__(self):
- return self._lock.__enter__()
-
- def __exit__(self, *args):
- return self._lock.__exit__(*args)
-
- def __reduce__(self):
- assert_spawning(self)
- return synchronized, (self._obj, self._lock)
-
- def get_obj(self):
- return self._obj
-
- def get_lock(self):
- return self._lock
-
- def __repr__(self):
- return '<%s wrapper for %s>' % (type(self).__name__, self._obj)
-
-
-class Synchronized(SynchronizedBase):
- value = make_property('value')
-
-
-class SynchronizedArray(SynchronizedBase):
-
- def __len__(self):
- return len(self._obj)
-
- def __getitem__(self, i):
- with self:
- return self._obj[i]
-
- def __setitem__(self, i, value):
- with self:
- self._obj[i] = value
-
- def __getslice__(self, start, stop):
- with self:
- return self._obj[start:stop]
-
- def __setslice__(self, start, stop, values):
- with self:
- self._obj[start:stop] = values
-
-
-class SynchronizedString(SynchronizedArray):
- value = make_property('value')
- raw = make_property('raw')
+#
+# Module which supports allocation of ctypes objects from shared memory
+#
+# multiprocessing/sharedctypes.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import ctypes
+import weakref
+
+from . import heap
+from . import get_context
+
+from .context import reduction, assert_spawning
+_ForkingPickler = reduction.ForkingPickler
+
+__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
+
+#
+#
+#
+
+typecode_to_type = {
+ 'c': ctypes.c_char, 'u': ctypes.c_wchar,
+ 'b': ctypes.c_byte, 'B': ctypes.c_ubyte,
+ 'h': ctypes.c_short, 'H': ctypes.c_ushort,
+ 'i': ctypes.c_int, 'I': ctypes.c_uint,
+ 'l': ctypes.c_long, 'L': ctypes.c_ulong,
+ 'q': ctypes.c_longlong, 'Q': ctypes.c_ulonglong,
+ 'f': ctypes.c_float, 'd': ctypes.c_double
+ }
+
+#
+#
+#
+
+def _new_value(type_):
+ size = ctypes.sizeof(type_)
+ wrapper = heap.BufferWrapper(size)
+ return rebuild_ctype(type_, wrapper, None)
+
+def RawValue(typecode_or_type, *args):
+ '''
+ Returns a ctypes object allocated from shared memory
+ '''
+ type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
+ obj = _new_value(type_)
+ ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+ obj.__init__(*args)
+ return obj
+
+def RawArray(typecode_or_type, size_or_initializer):
+ '''
+ Returns a ctypes array allocated from shared memory
+ '''
+ type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
+ if isinstance(size_or_initializer, int):
+ type_ = type_ * size_or_initializer
+ obj = _new_value(type_)
+ ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+ return obj
+ else:
+ type_ = type_ * len(size_or_initializer)
+ result = _new_value(type_)
+ result.__init__(*size_or_initializer)
+ return result
+
+def Value(typecode_or_type, *args, lock=True, ctx=None):
+ '''
+ Return a synchronization wrapper for a Value
+ '''
+ obj = RawValue(typecode_or_type, *args)
+ if lock is False:
+ return obj
+ if lock in (True, None):
+ ctx = ctx or get_context()
+ lock = ctx.RLock()
+ if not hasattr(lock, 'acquire'):
+ raise AttributeError("%r has no method 'acquire'" % lock)
+ return synchronized(obj, lock, ctx=ctx)
+
+def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None):
+ '''
+ Return a synchronization wrapper for a RawArray
+ '''
+ obj = RawArray(typecode_or_type, size_or_initializer)
+ if lock is False:
+ return obj
+ if lock in (True, None):
+ ctx = ctx or get_context()
+ lock = ctx.RLock()
+ if not hasattr(lock, 'acquire'):
+ raise AttributeError("%r has no method 'acquire'" % lock)
+ return synchronized(obj, lock, ctx=ctx)
+
+def copy(obj):
+ new_obj = _new_value(type(obj))
+ ctypes.pointer(new_obj)[0] = obj
+ return new_obj
+
+def synchronized(obj, lock=None, ctx=None):
+ assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
+ ctx = ctx or get_context()
+
+ if isinstance(obj, ctypes._SimpleCData):
+ return Synchronized(obj, lock, ctx)
+ elif isinstance(obj, ctypes.Array):
+ if obj._type_ is ctypes.c_char:
+ return SynchronizedString(obj, lock, ctx)
+ return SynchronizedArray(obj, lock, ctx)
+ else:
+ cls = type(obj)
+ try:
+ scls = class_cache[cls]
+ except KeyError:
+ names = [field[0] for field in cls._fields_]
+ d = {name: make_property(name) for name in names}
+ classname = 'Synchronized' + cls.__name__
+ scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
+ return scls(obj, lock, ctx)
+
+#
+# Functions for pickling/unpickling
+#
+
+def reduce_ctype(obj):
+ assert_spawning(obj)
+ if isinstance(obj, ctypes.Array):
+ return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)
+ else:
+ return rebuild_ctype, (type(obj), obj._wrapper, None)
+
+def rebuild_ctype(type_, wrapper, length):
+ if length is not None:
+ type_ = type_ * length
+ _ForkingPickler.register(type_, reduce_ctype)
+ buf = wrapper.create_memoryview()
+ obj = type_.from_buffer(buf)
+ obj._wrapper = wrapper
+ return obj
+
+#
+# Function to create properties
+#
+
+def make_property(name):
+ try:
+ return prop_cache[name]
+ except KeyError:
+ d = {}
+ exec(template % ((name,)*7), d)
+ prop_cache[name] = d[name]
+ return d[name]
+
+template = '''
+def get%s(self):
+ self.acquire()
+ try:
+ return self._obj.%s
+ finally:
+ self.release()
+def set%s(self, value):
+ self.acquire()
+ try:
+ self._obj.%s = value
+ finally:
+ self.release()
+%s = property(get%s, set%s)
+'''
+
+prop_cache = {}
+class_cache = weakref.WeakKeyDictionary()
+
+#
+# Synchronized wrappers
+#
+
+class SynchronizedBase(object):
+
+ def __init__(self, obj, lock=None, ctx=None):
+ self._obj = obj
+ if lock:
+ self._lock = lock
+ else:
+ ctx = ctx or get_context(force=True)
+ self._lock = ctx.RLock()
+ self.acquire = self._lock.acquire
+ self.release = self._lock.release
+
+ def __enter__(self):
+ return self._lock.__enter__()
+
+ def __exit__(self, *args):
+ return self._lock.__exit__(*args)
+
+ def __reduce__(self):
+ assert_spawning(self)
+ return synchronized, (self._obj, self._lock)
+
+ def get_obj(self):
+ return self._obj
+
+ def get_lock(self):
+ return self._lock
+
+ def __repr__(self):
+ return '<%s wrapper for %s>' % (type(self).__name__, self._obj)
+
+
+class Synchronized(SynchronizedBase):
+ value = make_property('value')
+
+
+class SynchronizedArray(SynchronizedBase):
+
+ def __len__(self):
+ return len(self._obj)
+
+ def __getitem__(self, i):
+ with self:
+ return self._obj[i]
+
+ def __setitem__(self, i, value):
+ with self:
+ self._obj[i] = value
+
+ def __getslice__(self, start, stop):
+ with self:
+ return self._obj[start:stop]
+
+ def __setslice__(self, start, stop, values):
+ with self:
+ self._obj[start:stop] = values
+
+
+class SynchronizedString(SynchronizedArray):
+ value = make_property('value')
+ raw = make_property('raw')
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/spawn.py b/contrib/tools/python3/src/Lib/multiprocessing/spawn.py
index f7f8deb2463..5787d371b58 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/spawn.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/spawn.py
@@ -1,101 +1,101 @@
-#
-# Code used to start processes when using the spawn or forkserver
-# start methods.
-#
-# multiprocessing/spawn.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-import os
-import sys
-import runpy
-import types
-
-from . import get_start_method, set_start_method
-from . import process
-from .context import reduction
-from . import util
-
-__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
- 'get_preparation_data', 'get_command_line', 'import_main_path']
-
-#
-# _python_exe is the assumed path to the python executable.
-# People embedding Python want to modify it.
-#
-
-if sys.platform != 'win32':
- WINEXE = False
- WINSERVICE = False
-else:
- WINEXE = getattr(sys, 'frozen', False)
- WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
-
-if WINSERVICE:
- _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
-else:
- _python_exe = sys.executable
-
-def set_executable(exe):
- global _python_exe
- _python_exe = exe
-
-def get_executable():
- return _python_exe
-
-#
-#
-#
-
-def is_forking(argv):
- '''
- Return whether commandline indicates we are forking
- '''
- if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
- return True
- else:
- return False
-
-
-def freeze_support():
- '''
- Run code for process object if this in not the main process
- '''
- if is_forking(sys.argv):
- kwds = {}
- for arg in sys.argv[2:]:
- name, value = arg.split('=')
- if value == 'None':
- kwds[name] = None
- else:
- kwds[name] = int(value)
- spawn_main(**kwds)
- sys.exit()
-
-
-def get_command_line(**kwds):
- '''
- Returns prefix of command line used for spawning a child process
- '''
- if False and getattr(sys, 'frozen', False):
- return ([sys.executable, '--multiprocessing-fork'] +
- ['%s=%r' % item for item in kwds.items()])
- else:
- prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
- prog %= ', '.join('%s=%r' % item for item in kwds.items())
- opts = util._args_from_interpreter_flags()
- return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
-
-
-def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
- '''
- Run code specified by data received over pipe
- '''
- assert is_forking(sys.argv), "Not forking"
- if sys.platform == 'win32':
- import msvcrt
+#
+# Code used to start processes when using the spawn or forkserver
+# start methods.
+#
+# multiprocessing/spawn.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import os
+import sys
+import runpy
+import types
+
+from . import get_start_method, set_start_method
+from . import process
+from .context import reduction
+from . import util
+
+__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
+ 'get_preparation_data', 'get_command_line', 'import_main_path']
+
+#
+# _python_exe is the assumed path to the python executable.
+# People embedding Python want to modify it.
+#
+
+if sys.platform != 'win32':
+ WINEXE = False
+ WINSERVICE = False
+else:
+ WINEXE = getattr(sys, 'frozen', False)
+ WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
+
+if WINSERVICE:
+ _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
+else:
+ _python_exe = sys.executable
+
+def set_executable(exe):
+ global _python_exe
+ _python_exe = exe
+
+def get_executable():
+ return _python_exe
+
+#
+#
+#
+
+def is_forking(argv):
+ '''
+ Return whether commandline indicates we are forking
+ '''
+ if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
+ return True
+ else:
+ return False
+
+
+def freeze_support():
+ '''
+ Run code for process object if this in not the main process
+ '''
+ if is_forking(sys.argv):
+ kwds = {}
+ for arg in sys.argv[2:]:
+ name, value = arg.split('=')
+ if value == 'None':
+ kwds[name] = None
+ else:
+ kwds[name] = int(value)
+ spawn_main(**kwds)
+ sys.exit()
+
+
+def get_command_line(**kwds):
+ '''
+ Returns prefix of command line used for spawning a child process
+ '''
+ if False and getattr(sys, 'frozen', False):
+ return ([sys.executable, '--multiprocessing-fork'] +
+ ['%s=%r' % item for item in kwds.items()])
+ else:
+ prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
+ prog %= ', '.join('%s=%r' % item for item in kwds.items())
+ opts = util._args_from_interpreter_flags()
+ return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
+
+
+def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
+ '''
+ Run code specified by data received over pipe
+ '''
+ assert is_forking(sys.argv), "Not forking"
+ if sys.platform == 'win32':
+ import msvcrt
import _winapi
if parent_pid is not None:
@@ -106,192 +106,192 @@ def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
source_process = None
new_handle = reduction.duplicate(pipe_handle,
source_process=source_process)
- fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
+ fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
parent_sentinel = source_process
- else:
+ else:
from . import resource_tracker
resource_tracker._resource_tracker._fd = tracker_fd
- fd = pipe_handle
+ fd = pipe_handle
parent_sentinel = os.dup(pipe_handle)
exitcode = _main(fd, parent_sentinel)
- sys.exit(exitcode)
-
-
+ sys.exit(exitcode)
+
+
def _main(fd, parent_sentinel):
- with os.fdopen(fd, 'rb', closefd=True) as from_parent:
- process.current_process()._inheriting = True
- try:
- preparation_data = reduction.pickle.load(from_parent)
- prepare(preparation_data)
- self = reduction.pickle.load(from_parent)
- finally:
- del process.current_process()._inheriting
+ with os.fdopen(fd, 'rb', closefd=True) as from_parent:
+ process.current_process()._inheriting = True
+ try:
+ preparation_data = reduction.pickle.load(from_parent)
+ prepare(preparation_data)
+ self = reduction.pickle.load(from_parent)
+ finally:
+ del process.current_process()._inheriting
return self._bootstrap(parent_sentinel)
-
-
-def _check_not_importing_main():
- if getattr(process.current_process(), '_inheriting', False):
- raise RuntimeError('''
- An attempt has been made to start a new process before the
- current process has finished its bootstrapping phase.
-
- This probably means that you are not using fork to start your
- child processes and you have forgotten to use the proper idiom
- in the main module:
-
- if __name__ == '__main__':
- freeze_support()
- ...
-
- The "freeze_support()" line can be omitted if the program
- is not going to be frozen to produce an executable.''')
-
-
-def get_preparation_data(name):
- '''
- Return info about parent needed by child to unpickle process object
- '''
- _check_not_importing_main()
- d = dict(
- log_to_stderr=util._log_to_stderr,
- authkey=process.current_process().authkey,
- )
-
- if util._logger is not None:
- d['log_level'] = util._logger.getEffectiveLevel()
-
- sys_path=sys.path.copy()
- try:
- i = sys_path.index('')
- except ValueError:
- pass
- else:
- sys_path[i] = process.ORIGINAL_DIR
-
- d.update(
- name=name,
- sys_path=sys_path,
- sys_argv=sys.argv,
- orig_dir=process.ORIGINAL_DIR,
- dir=os.getcwd(),
- start_method=get_start_method(),
- )
-
- # Figure out whether to initialise main in the subprocess as a module
- # or through direct execution (or to leave it alone entirely)
- main_module = sys.modules['__main__']
- main_mod_name = getattr(main_module.__spec__, "name", None)
- if main_mod_name is not None:
- d['init_main_from_name'] = main_mod_name
- elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
- main_path = getattr(main_module, '__file__', None)
- if main_path is not None:
- if (not os.path.isabs(main_path) and
- process.ORIGINAL_DIR is not None):
- main_path = os.path.join(process.ORIGINAL_DIR, main_path)
- d['init_main_from_path'] = os.path.normpath(main_path)
-
- return d
-
-#
-# Prepare current process
-#
-
-old_main_modules = []
-
-def prepare(data):
- '''
- Try to get current process ready to unpickle process object
- '''
- if 'name' in data:
- process.current_process().name = data['name']
-
- if 'authkey' in data:
- process.current_process().authkey = data['authkey']
-
- if 'log_to_stderr' in data and data['log_to_stderr']:
- util.log_to_stderr()
-
- if 'log_level' in data:
- util.get_logger().setLevel(data['log_level'])
-
- if 'sys_path' in data:
- sys.path = data['sys_path']
-
- if 'sys_argv' in data:
- sys.argv = data['sys_argv']
-
- if 'dir' in data:
- os.chdir(data['dir'])
-
- if 'orig_dir' in data:
- process.ORIGINAL_DIR = data['orig_dir']
-
- if 'start_method' in data:
- set_start_method(data['start_method'], force=True)
-
- if 'init_main_from_name' in data:
- _fixup_main_from_name(data['init_main_from_name'])
- elif 'init_main_from_path' in data:
- _fixup_main_from_path(data['init_main_from_path'])
-
-# Multiprocessing module helpers to fix up the main module in
-# spawned subprocesses
-def _fixup_main_from_name(mod_name):
- # __main__.py files for packages, directories, zip archives, etc, run
- # their "main only" code unconditionally, so we don't even try to
- # populate anything in __main__, nor do we make any changes to
- # __main__ attributes
- current_main = sys.modules['__main__']
- if mod_name == "__main__" or mod_name.endswith(".__main__"):
- return
-
- # If this process was forked, __main__ may already be populated
- if getattr(current_main.__spec__, "name", None) == mod_name:
- return
-
- # Otherwise, __main__ may contain some non-main code where we need to
- # support unpickling it properly. We rerun it as __mp_main__ and make
- # the normal __main__ an alias to that
- old_main_modules.append(current_main)
- main_module = types.ModuleType("__mp_main__")
- main_content = runpy.run_module(mod_name,
- run_name="__mp_main__",
- alter_sys=True)
- main_module.__dict__.update(main_content)
- sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
-
-
-def _fixup_main_from_path(main_path):
- # If this process was forked, __main__ may already be populated
- current_main = sys.modules['__main__']
-
- # Unfortunately, the main ipython launch script historically had no
- # "if __name__ == '__main__'" guard, so we work around that
- # by treating it like a __main__.py file
- # See https://github.com/ipython/ipython/issues/4698
- main_name = os.path.splitext(os.path.basename(main_path))[0]
- if main_name == 'ipython':
- return
-
- # Otherwise, if __file__ already has the setting we expect,
- # there's nothing more to do
- if getattr(current_main, '__file__', None) == main_path:
- return
-
- # If the parent process has sent a path through rather than a module
- # name we assume it is an executable script that may contain
- # non-main code that needs to be executed
- old_main_modules.append(current_main)
- main_module = types.ModuleType("__mp_main__")
- main_content = runpy.run_path(main_path,
- run_name="__mp_main__")
- main_module.__dict__.update(main_content)
- sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
-
-
-def import_main_path(main_path):
- '''
- Set sys.modules['__main__'] to module at main_path
- '''
- _fixup_main_from_path(main_path)
+
+
+def _check_not_importing_main():
+ if getattr(process.current_process(), '_inheriting', False):
+ raise RuntimeError('''
+ An attempt has been made to start a new process before the
+ current process has finished its bootstrapping phase.
+
+ This probably means that you are not using fork to start your
+ child processes and you have forgotten to use the proper idiom
+ in the main module:
+
+ if __name__ == '__main__':
+ freeze_support()
+ ...
+
+ The "freeze_support()" line can be omitted if the program
+ is not going to be frozen to produce an executable.''')
+
+
+def get_preparation_data(name):
+ '''
+ Return info about parent needed by child to unpickle process object
+ '''
+ _check_not_importing_main()
+ d = dict(
+ log_to_stderr=util._log_to_stderr,
+ authkey=process.current_process().authkey,
+ )
+
+ if util._logger is not None:
+ d['log_level'] = util._logger.getEffectiveLevel()
+
+ sys_path=sys.path.copy()
+ try:
+ i = sys_path.index('')
+ except ValueError:
+ pass
+ else:
+ sys_path[i] = process.ORIGINAL_DIR
+
+ d.update(
+ name=name,
+ sys_path=sys_path,
+ sys_argv=sys.argv,
+ orig_dir=process.ORIGINAL_DIR,
+ dir=os.getcwd(),
+ start_method=get_start_method(),
+ )
+
+ # Figure out whether to initialise main in the subprocess as a module
+ # or through direct execution (or to leave it alone entirely)
+ main_module = sys.modules['__main__']
+ main_mod_name = getattr(main_module.__spec__, "name", None)
+ if main_mod_name is not None:
+ d['init_main_from_name'] = main_mod_name
+ elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
+ main_path = getattr(main_module, '__file__', None)
+ if main_path is not None:
+ if (not os.path.isabs(main_path) and
+ process.ORIGINAL_DIR is not None):
+ main_path = os.path.join(process.ORIGINAL_DIR, main_path)
+ d['init_main_from_path'] = os.path.normpath(main_path)
+
+ return d
+
+#
+# Prepare current process
+#
+
+old_main_modules = []
+
+def prepare(data):
+ '''
+ Try to get current process ready to unpickle process object
+ '''
+ if 'name' in data:
+ process.current_process().name = data['name']
+
+ if 'authkey' in data:
+ process.current_process().authkey = data['authkey']
+
+ if 'log_to_stderr' in data and data['log_to_stderr']:
+ util.log_to_stderr()
+
+ if 'log_level' in data:
+ util.get_logger().setLevel(data['log_level'])
+
+ if 'sys_path' in data:
+ sys.path = data['sys_path']
+
+ if 'sys_argv' in data:
+ sys.argv = data['sys_argv']
+
+ if 'dir' in data:
+ os.chdir(data['dir'])
+
+ if 'orig_dir' in data:
+ process.ORIGINAL_DIR = data['orig_dir']
+
+ if 'start_method' in data:
+ set_start_method(data['start_method'], force=True)
+
+ if 'init_main_from_name' in data:
+ _fixup_main_from_name(data['init_main_from_name'])
+ elif 'init_main_from_path' in data:
+ _fixup_main_from_path(data['init_main_from_path'])
+
+# Multiprocessing module helpers to fix up the main module in
+# spawned subprocesses
+def _fixup_main_from_name(mod_name):
+ # __main__.py files for packages, directories, zip archives, etc, run
+ # their "main only" code unconditionally, so we don't even try to
+ # populate anything in __main__, nor do we make any changes to
+ # __main__ attributes
+ current_main = sys.modules['__main__']
+ if mod_name == "__main__" or mod_name.endswith(".__main__"):
+ return
+
+ # If this process was forked, __main__ may already be populated
+ if getattr(current_main.__spec__, "name", None) == mod_name:
+ return
+
+ # Otherwise, __main__ may contain some non-main code where we need to
+ # support unpickling it properly. We rerun it as __mp_main__ and make
+ # the normal __main__ an alias to that
+ old_main_modules.append(current_main)
+ main_module = types.ModuleType("__mp_main__")
+ main_content = runpy.run_module(mod_name,
+ run_name="__mp_main__",
+ alter_sys=True)
+ main_module.__dict__.update(main_content)
+ sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
+
+
+def _fixup_main_from_path(main_path):
+ # If this process was forked, __main__ may already be populated
+ current_main = sys.modules['__main__']
+
+ # Unfortunately, the main ipython launch script historically had no
+ # "if __name__ == '__main__'" guard, so we work around that
+ # by treating it like a __main__.py file
+ # See https://github.com/ipython/ipython/issues/4698
+ main_name = os.path.splitext(os.path.basename(main_path))[0]
+ if main_name == 'ipython':
+ return
+
+ # Otherwise, if __file__ already has the setting we expect,
+ # there's nothing more to do
+ if getattr(current_main, '__file__', None) == main_path:
+ return
+
+ # If the parent process has sent a path through rather than a module
+ # name we assume it is an executable script that may contain
+ # non-main code that needs to be executed
+ old_main_modules.append(current_main)
+ main_module = types.ModuleType("__mp_main__")
+ main_content = runpy.run_path(main_path,
+ run_name="__mp_main__")
+ main_module.__dict__.update(main_content)
+ sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
+
+
+def import_main_path(main_path):
+ '''
+ Set sys.modules['__main__'] to module at main_path
+ '''
+ _fixup_main_from_path(main_path)
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/synchronize.py b/contrib/tools/python3/src/Lib/multiprocessing/synchronize.py
index d0be48f1fd7..cc475c3389f 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/synchronize.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/synchronize.py
@@ -1,394 +1,394 @@
-#
-# Module implementing synchronization primitives
-#
-# multiprocessing/synchronize.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-__all__ = [
- 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'
- ]
-
-import threading
-import sys
-import tempfile
-import _multiprocessing
-import time
-
-from . import context
-from . import process
-from . import util
-
-# Try to import the mp.synchronize module cleanly, if it fails
-# raise ImportError for platforms lacking a working sem_open implementation.
-# See issue 3770
-try:
- from _multiprocessing import SemLock, sem_unlink
-except (ImportError):
- raise ImportError("This platform lacks a functioning sem_open" +
- " implementation, therefore, the required" +
- " synchronization primitives needed will not" +
- " function, see issue 3770.")
-
-#
-# Constants
-#
-
-RECURSIVE_MUTEX, SEMAPHORE = list(range(2))
-SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
-
-#
-# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`
-#
-
-class SemLock(object):
-
- _rand = tempfile._RandomNameSequence()
-
- def __init__(self, kind, value, maxvalue, *, ctx):
- if ctx is None:
- ctx = context._default_context.get_context()
- name = ctx.get_start_method()
- unlink_now = sys.platform == 'win32' or name == 'fork'
- for i in range(100):
- try:
- sl = self._semlock = _multiprocessing.SemLock(
- kind, value, maxvalue, self._make_name(),
- unlink_now)
- except FileExistsError:
- pass
- else:
- break
- else:
- raise FileExistsError('cannot find name for semaphore')
-
- util.debug('created semlock with handle %s' % sl.handle)
- self._make_methods()
-
- if sys.platform != 'win32':
- def _after_fork(obj):
- obj._semlock._after_fork()
- util.register_after_fork(self, _after_fork)
-
- if self._semlock.name is not None:
- # We only get here if we are on Unix with forking
- # disabled. When the object is garbage collected or the
- # process shuts down we unlink the semaphore name
+#
+# Module implementing synchronization primitives
+#
+# multiprocessing/synchronize.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = [
+ 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'
+ ]
+
+import threading
+import sys
+import tempfile
+import _multiprocessing
+import time
+
+from . import context
+from . import process
+from . import util
+
+# Try to import the mp.synchronize module cleanly, if it fails
+# raise ImportError for platforms lacking a working sem_open implementation.
+# See issue 3770
+try:
+ from _multiprocessing import SemLock, sem_unlink
+except (ImportError):
+ raise ImportError("This platform lacks a functioning sem_open" +
+ " implementation, therefore, the required" +
+ " synchronization primitives needed will not" +
+ " function, see issue 3770.")
+
+#
+# Constants
+#
+
+RECURSIVE_MUTEX, SEMAPHORE = list(range(2))
+SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
+
+#
+# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`
+#
+
+class SemLock(object):
+
+ _rand = tempfile._RandomNameSequence()
+
+ def __init__(self, kind, value, maxvalue, *, ctx):
+ if ctx is None:
+ ctx = context._default_context.get_context()
+ name = ctx.get_start_method()
+ unlink_now = sys.platform == 'win32' or name == 'fork'
+ for i in range(100):
+ try:
+ sl = self._semlock = _multiprocessing.SemLock(
+ kind, value, maxvalue, self._make_name(),
+ unlink_now)
+ except FileExistsError:
+ pass
+ else:
+ break
+ else:
+ raise FileExistsError('cannot find name for semaphore')
+
+ util.debug('created semlock with handle %s' % sl.handle)
+ self._make_methods()
+
+ if sys.platform != 'win32':
+ def _after_fork(obj):
+ obj._semlock._after_fork()
+ util.register_after_fork(self, _after_fork)
+
+ if self._semlock.name is not None:
+ # We only get here if we are on Unix with forking
+ # disabled. When the object is garbage collected or the
+ # process shuts down we unlink the semaphore name
from .resource_tracker import register
register(self._semlock.name, "semaphore")
- util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
- exitpriority=0)
-
- @staticmethod
- def _cleanup(name):
+ util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
+ exitpriority=0)
+
+ @staticmethod
+ def _cleanup(name):
from .resource_tracker import unregister
- sem_unlink(name)
+ sem_unlink(name)
unregister(name, "semaphore")
-
- def _make_methods(self):
- self.acquire = self._semlock.acquire
- self.release = self._semlock.release
-
- def __enter__(self):
- return self._semlock.__enter__()
-
- def __exit__(self, *args):
- return self._semlock.__exit__(*args)
-
- def __getstate__(self):
- context.assert_spawning(self)
- sl = self._semlock
- if sys.platform == 'win32':
- h = context.get_spawning_popen().duplicate_for_child(sl.handle)
- else:
- h = sl.handle
- return (h, sl.kind, sl.maxvalue, sl.name)
-
- def __setstate__(self, state):
- self._semlock = _multiprocessing.SemLock._rebuild(*state)
- util.debug('recreated blocker with handle %r' % state[0])
- self._make_methods()
-
- @staticmethod
- def _make_name():
- return '%s-%s' % (process.current_process()._config['semprefix'],
- next(SemLock._rand))
-
-#
-# Semaphore
-#
-
-class Semaphore(SemLock):
-
- def __init__(self, value=1, *, ctx):
- SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
-
- def get_value(self):
- return self._semlock._get_value()
-
- def __repr__(self):
- try:
- value = self._semlock._get_value()
- except Exception:
- value = 'unknown'
- return '<%s(value=%s)>' % (self.__class__.__name__, value)
-
-#
-# Bounded semaphore
-#
-
-class BoundedSemaphore(Semaphore):
-
- def __init__(self, value=1, *, ctx):
- SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
-
- def __repr__(self):
- try:
- value = self._semlock._get_value()
- except Exception:
- value = 'unknown'
- return '<%s(value=%s, maxvalue=%s)>' % \
- (self.__class__.__name__, value, self._semlock.maxvalue)
-
-#
-# Non-recursive lock
-#
-
-class Lock(SemLock):
-
- def __init__(self, *, ctx):
- SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
-
- def __repr__(self):
- try:
- if self._semlock._is_mine():
- name = process.current_process().name
- if threading.current_thread().name != 'MainThread':
- name += '|' + threading.current_thread().name
- elif self._semlock._get_value() == 1:
- name = 'None'
- elif self._semlock._count() > 0:
- name = 'SomeOtherThread'
- else:
- name = 'SomeOtherProcess'
- except Exception:
- name = 'unknown'
- return '<%s(owner=%s)>' % (self.__class__.__name__, name)
-
-#
-# Recursive lock
-#
-
-class RLock(SemLock):
-
- def __init__(self, *, ctx):
- SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
-
- def __repr__(self):
- try:
- if self._semlock._is_mine():
- name = process.current_process().name
- if threading.current_thread().name != 'MainThread':
- name += '|' + threading.current_thread().name
- count = self._semlock._count()
- elif self._semlock._get_value() == 1:
- name, count = 'None', 0
- elif self._semlock._count() > 0:
- name, count = 'SomeOtherThread', 'nonzero'
- else:
- name, count = 'SomeOtherProcess', 'nonzero'
- except Exception:
- name, count = 'unknown', 'unknown'
- return '<%s(%s, %s)>' % (self.__class__.__name__, name, count)
-
-#
-# Condition variable
-#
-
-class Condition(object):
-
- def __init__(self, lock=None, *, ctx):
- self._lock = lock or ctx.RLock()
- self._sleeping_count = ctx.Semaphore(0)
- self._woken_count = ctx.Semaphore(0)
- self._wait_semaphore = ctx.Semaphore(0)
- self._make_methods()
-
- def __getstate__(self):
- context.assert_spawning(self)
- return (self._lock, self._sleeping_count,
- self._woken_count, self._wait_semaphore)
-
- def __setstate__(self, state):
- (self._lock, self._sleeping_count,
- self._woken_count, self._wait_semaphore) = state
- self._make_methods()
-
- def __enter__(self):
- return self._lock.__enter__()
-
- def __exit__(self, *args):
- return self._lock.__exit__(*args)
-
- def _make_methods(self):
- self.acquire = self._lock.acquire
- self.release = self._lock.release
-
- def __repr__(self):
- try:
- num_waiters = (self._sleeping_count._semlock._get_value() -
- self._woken_count._semlock._get_value())
- except Exception:
- num_waiters = 'unknown'
- return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters)
-
- def wait(self, timeout=None):
- assert self._lock._semlock._is_mine(), \
- 'must acquire() condition before using wait()'
-
- # indicate that this thread is going to sleep
- self._sleeping_count.release()
-
- # release lock
- count = self._lock._semlock._count()
- for i in range(count):
- self._lock.release()
-
- try:
- # wait for notification or timeout
- return self._wait_semaphore.acquire(True, timeout)
- finally:
- # indicate that this thread has woken
- self._woken_count.release()
-
- # reacquire lock
- for i in range(count):
- self._lock.acquire()
-
- def notify(self, n=1):
- assert self._lock._semlock._is_mine(), 'lock is not owned'
- assert not self._wait_semaphore.acquire(
+
+ def _make_methods(self):
+ self.acquire = self._semlock.acquire
+ self.release = self._semlock.release
+
+ def __enter__(self):
+ return self._semlock.__enter__()
+
+ def __exit__(self, *args):
+ return self._semlock.__exit__(*args)
+
+ def __getstate__(self):
+ context.assert_spawning(self)
+ sl = self._semlock
+ if sys.platform == 'win32':
+ h = context.get_spawning_popen().duplicate_for_child(sl.handle)
+ else:
+ h = sl.handle
+ return (h, sl.kind, sl.maxvalue, sl.name)
+
+ def __setstate__(self, state):
+ self._semlock = _multiprocessing.SemLock._rebuild(*state)
+ util.debug('recreated blocker with handle %r' % state[0])
+ self._make_methods()
+
+ @staticmethod
+ def _make_name():
+ return '%s-%s' % (process.current_process()._config['semprefix'],
+ next(SemLock._rand))
+
+#
+# Semaphore
+#
+
+class Semaphore(SemLock):
+
+ def __init__(self, value=1, *, ctx):
+ SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
+
+ def get_value(self):
+ return self._semlock._get_value()
+
+ def __repr__(self):
+ try:
+ value = self._semlock._get_value()
+ except Exception:
+ value = 'unknown'
+ return '<%s(value=%s)>' % (self.__class__.__name__, value)
+
+#
+# Bounded semaphore
+#
+
+class BoundedSemaphore(Semaphore):
+
+ def __init__(self, value=1, *, ctx):
+ SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
+
+ def __repr__(self):
+ try:
+ value = self._semlock._get_value()
+ except Exception:
+ value = 'unknown'
+ return '<%s(value=%s, maxvalue=%s)>' % \
+ (self.__class__.__name__, value, self._semlock.maxvalue)
+
+#
+# Non-recursive lock
+#
+
+class Lock(SemLock):
+
+ def __init__(self, *, ctx):
+ SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
+
+ def __repr__(self):
+ try:
+ if self._semlock._is_mine():
+ name = process.current_process().name
+ if threading.current_thread().name != 'MainThread':
+ name += '|' + threading.current_thread().name
+ elif self._semlock._get_value() == 1:
+ name = 'None'
+ elif self._semlock._count() > 0:
+ name = 'SomeOtherThread'
+ else:
+ name = 'SomeOtherProcess'
+ except Exception:
+ name = 'unknown'
+ return '<%s(owner=%s)>' % (self.__class__.__name__, name)
+
+#
+# Recursive lock
+#
+
+class RLock(SemLock):
+
+ def __init__(self, *, ctx):
+ SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
+
+ def __repr__(self):
+ try:
+ if self._semlock._is_mine():
+ name = process.current_process().name
+ if threading.current_thread().name != 'MainThread':
+ name += '|' + threading.current_thread().name
+ count = self._semlock._count()
+ elif self._semlock._get_value() == 1:
+ name, count = 'None', 0
+ elif self._semlock._count() > 0:
+ name, count = 'SomeOtherThread', 'nonzero'
+ else:
+ name, count = 'SomeOtherProcess', 'nonzero'
+ except Exception:
+ name, count = 'unknown', 'unknown'
+ return '<%s(%s, %s)>' % (self.__class__.__name__, name, count)
+
+#
+# Condition variable
+#
+
+class Condition(object):
+
+ def __init__(self, lock=None, *, ctx):
+ self._lock = lock or ctx.RLock()
+ self._sleeping_count = ctx.Semaphore(0)
+ self._woken_count = ctx.Semaphore(0)
+ self._wait_semaphore = ctx.Semaphore(0)
+ self._make_methods()
+
+ def __getstate__(self):
+ context.assert_spawning(self)
+ return (self._lock, self._sleeping_count,
+ self._woken_count, self._wait_semaphore)
+
+ def __setstate__(self, state):
+ (self._lock, self._sleeping_count,
+ self._woken_count, self._wait_semaphore) = state
+ self._make_methods()
+
+ def __enter__(self):
+ return self._lock.__enter__()
+
+ def __exit__(self, *args):
+ return self._lock.__exit__(*args)
+
+ def _make_methods(self):
+ self.acquire = self._lock.acquire
+ self.release = self._lock.release
+
+ def __repr__(self):
+ try:
+ num_waiters = (self._sleeping_count._semlock._get_value() -
+ self._woken_count._semlock._get_value())
+ except Exception:
+ num_waiters = 'unknown'
+ return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters)
+
+ def wait(self, timeout=None):
+ assert self._lock._semlock._is_mine(), \
+ 'must acquire() condition before using wait()'
+
+ # indicate that this thread is going to sleep
+ self._sleeping_count.release()
+
+ # release lock
+ count = self._lock._semlock._count()
+ for i in range(count):
+ self._lock.release()
+
+ try:
+ # wait for notification or timeout
+ return self._wait_semaphore.acquire(True, timeout)
+ finally:
+ # indicate that this thread has woken
+ self._woken_count.release()
+
+ # reacquire lock
+ for i in range(count):
+ self._lock.acquire()
+
+ def notify(self, n=1):
+ assert self._lock._semlock._is_mine(), 'lock is not owned'
+ assert not self._wait_semaphore.acquire(
False), ('notify: Should not have been able to acquire '
- + '_wait_semaphore')
-
- # to take account of timeouts since last notify*() we subtract
- # woken_count from sleeping_count and rezero woken_count
- while self._woken_count.acquire(False):
- res = self._sleeping_count.acquire(False)
- assert res, ('notify: Bug in sleeping_count.acquire'
- + '- res should not be False')
-
- sleepers = 0
- while sleepers < n and self._sleeping_count.acquire(False):
- self._wait_semaphore.release() # wake up one sleeper
- sleepers += 1
-
- if sleepers:
- for i in range(sleepers):
- self._woken_count.acquire() # wait for a sleeper to wake
-
- # rezero wait_semaphore in case some timeouts just happened
- while self._wait_semaphore.acquire(False):
- pass
-
- def notify_all(self):
- self.notify(n=sys.maxsize)
-
- def wait_for(self, predicate, timeout=None):
- result = predicate()
- if result:
- return result
- if timeout is not None:
- endtime = time.monotonic() + timeout
- else:
- endtime = None
- waittime = None
- while not result:
- if endtime is not None:
- waittime = endtime - time.monotonic()
- if waittime <= 0:
- break
- self.wait(waittime)
- result = predicate()
- return result
-
-#
-# Event
-#
-
-class Event(object):
-
- def __init__(self, *, ctx):
- self._cond = ctx.Condition(ctx.Lock())
- self._flag = ctx.Semaphore(0)
-
- def is_set(self):
- with self._cond:
- if self._flag.acquire(False):
- self._flag.release()
- return True
- return False
-
- def set(self):
- with self._cond:
- self._flag.acquire(False)
- self._flag.release()
- self._cond.notify_all()
-
- def clear(self):
- with self._cond:
- self._flag.acquire(False)
-
- def wait(self, timeout=None):
- with self._cond:
- if self._flag.acquire(False):
- self._flag.release()
- else:
- self._cond.wait(timeout)
-
- if self._flag.acquire(False):
- self._flag.release()
- return True
- return False
-
-#
-# Barrier
-#
-
-class Barrier(threading.Barrier):
-
- def __init__(self, parties, action=None, timeout=None, *, ctx):
- import struct
- from .heap import BufferWrapper
- wrapper = BufferWrapper(struct.calcsize('i') * 2)
- cond = ctx.Condition()
- self.__setstate__((parties, action, timeout, cond, wrapper))
- self._state = 0
- self._count = 0
-
- def __setstate__(self, state):
- (self._parties, self._action, self._timeout,
- self._cond, self._wrapper) = state
- self._array = self._wrapper.create_memoryview().cast('i')
-
- def __getstate__(self):
- return (self._parties, self._action, self._timeout,
- self._cond, self._wrapper)
-
- @property
- def _state(self):
- return self._array[0]
-
- @_state.setter
- def _state(self, value):
- self._array[0] = value
-
- @property
- def _count(self):
- return self._array[1]
-
- @_count.setter
- def _count(self, value):
- self._array[1] = value
+ + '_wait_semaphore')
+
+ # to take account of timeouts since last notify*() we subtract
+ # woken_count from sleeping_count and rezero woken_count
+ while self._woken_count.acquire(False):
+ res = self._sleeping_count.acquire(False)
+ assert res, ('notify: Bug in sleeping_count.acquire'
+ + '- res should not be False')
+
+ sleepers = 0
+ while sleepers < n and self._sleeping_count.acquire(False):
+ self._wait_semaphore.release() # wake up one sleeper
+ sleepers += 1
+
+ if sleepers:
+ for i in range(sleepers):
+ self._woken_count.acquire() # wait for a sleeper to wake
+
+ # rezero wait_semaphore in case some timeouts just happened
+ while self._wait_semaphore.acquire(False):
+ pass
+
+ def notify_all(self):
+ self.notify(n=sys.maxsize)
+
+ def wait_for(self, predicate, timeout=None):
+ result = predicate()
+ if result:
+ return result
+ if timeout is not None:
+ endtime = time.monotonic() + timeout
+ else:
+ endtime = None
+ waittime = None
+ while not result:
+ if endtime is not None:
+ waittime = endtime - time.monotonic()
+ if waittime <= 0:
+ break
+ self.wait(waittime)
+ result = predicate()
+ return result
+
+#
+# Event
+#
+
+class Event(object):
+
+ def __init__(self, *, ctx):
+ self._cond = ctx.Condition(ctx.Lock())
+ self._flag = ctx.Semaphore(0)
+
+ def is_set(self):
+ with self._cond:
+ if self._flag.acquire(False):
+ self._flag.release()
+ return True
+ return False
+
+ def set(self):
+ with self._cond:
+ self._flag.acquire(False)
+ self._flag.release()
+ self._cond.notify_all()
+
+ def clear(self):
+ with self._cond:
+ self._flag.acquire(False)
+
+ def wait(self, timeout=None):
+ with self._cond:
+ if self._flag.acquire(False):
+ self._flag.release()
+ else:
+ self._cond.wait(timeout)
+
+ if self._flag.acquire(False):
+ self._flag.release()
+ return True
+ return False
+
+#
+# Barrier
+#
+
+class Barrier(threading.Barrier):
+
+ def __init__(self, parties, action=None, timeout=None, *, ctx):
+ import struct
+ from .heap import BufferWrapper
+ wrapper = BufferWrapper(struct.calcsize('i') * 2)
+ cond = ctx.Condition()
+ self.__setstate__((parties, action, timeout, cond, wrapper))
+ self._state = 0
+ self._count = 0
+
+ def __setstate__(self, state):
+ (self._parties, self._action, self._timeout,
+ self._cond, self._wrapper) = state
+ self._array = self._wrapper.create_memoryview().cast('i')
+
+ def __getstate__(self):
+ return (self._parties, self._action, self._timeout,
+ self._cond, self._wrapper)
+
+ @property
+ def _state(self):
+ return self._array[0]
+
+ @_state.setter
+ def _state(self, value):
+ self._array[0] = value
+
+ @property
+ def _count(self):
+ return self._array[1]
+
+ @_count.setter
+ def _count(self, value):
+ self._array[1] = value
diff --git a/contrib/tools/python3/src/Lib/multiprocessing/util.py b/contrib/tools/python3/src/Lib/multiprocessing/util.py
index e94466be8ed..b9d556571a9 100644
--- a/contrib/tools/python3/src/Lib/multiprocessing/util.py
+++ b/contrib/tools/python3/src/Lib/multiprocessing/util.py
@@ -1,107 +1,107 @@
-#
-# Module providing various facilities to other parts of the package
-#
-# multiprocessing/util.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-import os
-import itertools
-import sys
-import weakref
-import atexit
-import threading # we want threading to install it's
- # cleanup function before multiprocessing does
-from subprocess import _args_from_interpreter_flags
-
-from . import process
-
-__all__ = [
- 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
- 'log_to_stderr', 'get_temp_dir', 'register_after_fork',
- 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
- 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING',
- ]
-
-#
-# Logging
-#
-
-NOTSET = 0
-SUBDEBUG = 5
-DEBUG = 10
-INFO = 20
-SUBWARNING = 25
-
-LOGGER_NAME = 'multiprocessing'
-DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'
-
-_logger = None
-_log_to_stderr = False
-
-def sub_debug(msg, *args):
- if _logger:
- _logger.log(SUBDEBUG, msg, *args)
-
-def debug(msg, *args):
- if _logger:
- _logger.log(DEBUG, msg, *args)
-
-def info(msg, *args):
- if _logger:
- _logger.log(INFO, msg, *args)
-
-def sub_warning(msg, *args):
- if _logger:
- _logger.log(SUBWARNING, msg, *args)
-
-def get_logger():
- '''
- Returns logger used by multiprocessing
- '''
- global _logger
- import logging
-
- logging._acquireLock()
- try:
- if not _logger:
-
- _logger = logging.getLogger(LOGGER_NAME)
- _logger.propagate = 0
-
- # XXX multiprocessing should cleanup before logging
- if hasattr(atexit, 'unregister'):
- atexit.unregister(_exit_function)
- atexit.register(_exit_function)
- else:
- atexit._exithandlers.remove((_exit_function, (), {}))
- atexit._exithandlers.append((_exit_function, (), {}))
-
- finally:
- logging._releaseLock()
-
- return _logger
-
-def log_to_stderr(level=None):
- '''
- Turn on logging and add a handler which prints to stderr
- '''
- global _log_to_stderr
- import logging
-
- logger = get_logger()
- formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
- handler = logging.StreamHandler()
- handler.setFormatter(formatter)
- logger.addHandler(handler)
-
- if level:
- logger.setLevel(level)
- _log_to_stderr = True
- return _logger
-
+#
+# Module providing various facilities to other parts of the package
+#
+# multiprocessing/util.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import os
+import itertools
+import sys
+import weakref
+import atexit
+import threading # we want threading to install it's
+ # cleanup function before multiprocessing does
+from subprocess import _args_from_interpreter_flags
+
+from . import process
+
+__all__ = [
+ 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
+ 'log_to_stderr', 'get_temp_dir', 'register_after_fork',
+ 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
+ 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING',
+ ]
+
+#
+# Logging
+#
+
+NOTSET = 0
+SUBDEBUG = 5
+DEBUG = 10
+INFO = 20
+SUBWARNING = 25
+
+LOGGER_NAME = 'multiprocessing'
+DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'
+
+_logger = None
+_log_to_stderr = False
+
+def sub_debug(msg, *args):
+ if _logger:
+ _logger.log(SUBDEBUG, msg, *args)
+
+def debug(msg, *args):
+ if _logger:
+ _logger.log(DEBUG, msg, *args)
+
+def info(msg, *args):
+ if _logger:
+ _logger.log(INFO, msg, *args)
+
+def sub_warning(msg, *args):
+ if _logger:
+ _logger.log(SUBWARNING, msg, *args)
+
+def get_logger():
+ '''
+ Returns logger used by multiprocessing
+ '''
+ global _logger
+ import logging
+
+ logging._acquireLock()
+ try:
+ if not _logger:
+
+ _logger = logging.getLogger(LOGGER_NAME)
+ _logger.propagate = 0
+
+ # XXX multiprocessing should cleanup before logging
+ if hasattr(atexit, 'unregister'):
+ atexit.unregister(_exit_function)
+ atexit.register(_exit_function)
+ else:
+ atexit._exithandlers.remove((_exit_function, (), {}))
+ atexit._exithandlers.append((_exit_function, (), {}))
+
+ finally:
+ logging._releaseLock()
+
+ return _logger
+
+def log_to_stderr(level=None):
+ '''
+ Turn on logging and add a handler which prints to stderr
+ '''
+ global _log_to_stderr
+ import logging
+
+ logger = get_logger()
+ formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+ handler = logging.StreamHandler()
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ if level:
+ logger.setLevel(level)
+ _log_to_stderr = True
+ return _logger
+
# Abstract socket support
@@ -125,10 +125,10 @@ def is_abstract_socket_namespace(address):
abstract_sockets_supported = _platform_supports_abstract_sockets()
-#
-# Function returning a temp directory which will be removed on exit
-#
-
+#
+# Function returning a temp directory which will be removed on exit
+#
+
def _remove_temp_dir(rmtree, tempdir):
rmtree(tempdir)
@@ -138,339 +138,339 @@ def _remove_temp_dir(rmtree, tempdir):
if current_process is not None:
current_process._config['tempdir'] = None
-def get_temp_dir():
- # get name of a temp directory which will be automatically cleaned up
- tempdir = process.current_process()._config.get('tempdir')
- if tempdir is None:
- import shutil, tempfile
- tempdir = tempfile.mkdtemp(prefix='pymp-')
- info('created temp directory %s', tempdir)
+def get_temp_dir():
+ # get name of a temp directory which will be automatically cleaned up
+ tempdir = process.current_process()._config.get('tempdir')
+ if tempdir is None:
+ import shutil, tempfile
+ tempdir = tempfile.mkdtemp(prefix='pymp-')
+ info('created temp directory %s', tempdir)
# keep a strong reference to shutil.rmtree(), since the finalizer
# can be called late during Python shutdown
Finalize(None, _remove_temp_dir, args=(shutil.rmtree, tempdir),
exitpriority=-100)
- process.current_process()._config['tempdir'] = tempdir
- return tempdir
-
-#
-# Support for reinitialization of objects when bootstrapping a child process
-#
-
-_afterfork_registry = weakref.WeakValueDictionary()
-_afterfork_counter = itertools.count()
-
-def _run_after_forkers():
- items = list(_afterfork_registry.items())
- items.sort()
- for (index, ident, func), obj in items:
- try:
- func(obj)
- except Exception as e:
- info('after forker raised exception %s', e)
-
-def register_after_fork(obj, func):
- _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj
-
-#
-# Finalization using weakrefs
-#
-
-_finalizer_registry = {}
-_finalizer_counter = itertools.count()
-
-
-class Finalize(object):
- '''
- Class which supports object finalization using weakrefs
- '''
- def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None):
- if (exitpriority is not None) and not isinstance(exitpriority,int):
- raise TypeError(
- "Exitpriority ({0!r}) must be None or int, not {1!s}".format(
- exitpriority, type(exitpriority)))
-
- if obj is not None:
- self._weakref = weakref.ref(obj, self)
- elif exitpriority is None:
- raise ValueError("Without object, exitpriority cannot be None")
-
- self._callback = callback
- self._args = args
- self._kwargs = kwargs or {}
- self._key = (exitpriority, next(_finalizer_counter))
- self._pid = os.getpid()
-
- _finalizer_registry[self._key] = self
-
- def __call__(self, wr=None,
- # Need to bind these locally because the globals can have
- # been cleared at shutdown
- _finalizer_registry=_finalizer_registry,
- sub_debug=sub_debug, getpid=os.getpid):
- '''
- Run the callback unless it has already been called or cancelled
- '''
- try:
- del _finalizer_registry[self._key]
- except KeyError:
- sub_debug('finalizer no longer registered')
- else:
- if self._pid != getpid():
- sub_debug('finalizer ignored because different process')
- res = None
- else:
- sub_debug('finalizer calling %s with args %s and kwargs %s',
- self._callback, self._args, self._kwargs)
- res = self._callback(*self._args, **self._kwargs)
- self._weakref = self._callback = self._args = \
- self._kwargs = self._key = None
- return res
-
- def cancel(self):
- '''
- Cancel finalization of the object
- '''
- try:
- del _finalizer_registry[self._key]
- except KeyError:
- pass
- else:
- self._weakref = self._callback = self._args = \
- self._kwargs = self._key = None
-
- def still_active(self):
- '''
- Return whether this finalizer is still waiting to invoke callback
- '''
- return self._key in _finalizer_registry
-
- def __repr__(self):
- try:
- obj = self._weakref()
- except (AttributeError, TypeError):
- obj = None
-
- if obj is None:
- return '<%s object, dead>' % self.__class__.__name__
-
- x = '<%s object, callback=%s' % (
- self.__class__.__name__,
- getattr(self._callback, '__name__', self._callback))
- if self._args:
- x += ', args=' + str(self._args)
- if self._kwargs:
- x += ', kwargs=' + str(self._kwargs)
- if self._key[0] is not None:
+ process.current_process()._config['tempdir'] = tempdir
+ return tempdir
+
+#
+# Support for reinitialization of objects when bootstrapping a child process
+#
+
+_afterfork_registry = weakref.WeakValueDictionary()
+_afterfork_counter = itertools.count()
+
+def _run_after_forkers():
+ items = list(_afterfork_registry.items())
+ items.sort()
+ for (index, ident, func), obj in items:
+ try:
+ func(obj)
+ except Exception as e:
+ info('after forker raised exception %s', e)
+
+def register_after_fork(obj, func):
+ _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj
+
+#
+# Finalization using weakrefs
+#
+
+_finalizer_registry = {}
+_finalizer_counter = itertools.count()
+
+
+class Finalize(object):
+ '''
+ Class which supports object finalization using weakrefs
+ '''
+ def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None):
+ if (exitpriority is not None) and not isinstance(exitpriority,int):
+ raise TypeError(
+ "Exitpriority ({0!r}) must be None or int, not {1!s}".format(
+ exitpriority, type(exitpriority)))
+
+ if obj is not None:
+ self._weakref = weakref.ref(obj, self)
+ elif exitpriority is None:
+ raise ValueError("Without object, exitpriority cannot be None")
+
+ self._callback = callback
+ self._args = args
+ self._kwargs = kwargs or {}
+ self._key = (exitpriority, next(_finalizer_counter))
+ self._pid = os.getpid()
+
+ _finalizer_registry[self._key] = self
+
+ def __call__(self, wr=None,
+ # Need to bind these locally because the globals can have
+ # been cleared at shutdown
+ _finalizer_registry=_finalizer_registry,
+ sub_debug=sub_debug, getpid=os.getpid):
+ '''
+ Run the callback unless it has already been called or cancelled
+ '''
+ try:
+ del _finalizer_registry[self._key]
+ except KeyError:
+ sub_debug('finalizer no longer registered')
+ else:
+ if self._pid != getpid():
+ sub_debug('finalizer ignored because different process')
+ res = None
+ else:
+ sub_debug('finalizer calling %s with args %s and kwargs %s',
+ self._callback, self._args, self._kwargs)
+ res = self._callback(*self._args, **self._kwargs)
+ self._weakref = self._callback = self._args = \
+ self._kwargs = self._key = None
+ return res
+
+ def cancel(self):
+ '''
+ Cancel finalization of the object
+ '''
+ try:
+ del _finalizer_registry[self._key]
+ except KeyError:
+ pass
+ else:
+ self._weakref = self._callback = self._args = \
+ self._kwargs = self._key = None
+
+ def still_active(self):
+ '''
+ Return whether this finalizer is still waiting to invoke callback
+ '''
+ return self._key in _finalizer_registry
+
+ def __repr__(self):
+ try:
+ obj = self._weakref()
+ except (AttributeError, TypeError):
+ obj = None
+
+ if obj is None:
+ return '<%s object, dead>' % self.__class__.__name__
+
+ x = '<%s object, callback=%s' % (
+ self.__class__.__name__,
+ getattr(self._callback, '__name__', self._callback))
+ if self._args:
+ x += ', args=' + str(self._args)
+ if self._kwargs:
+ x += ', kwargs=' + str(self._kwargs)
+ if self._key[0] is not None:
x += ', exitpriority=' + str(self._key[0])
- return x + '>'
-
-
-def _run_finalizers(minpriority=None):
- '''
- Run all finalizers whose exit priority is not None and at least minpriority
-
- Finalizers with highest priority are called first; finalizers with
- the same priority will be called in reverse order of creation.
- '''
- if _finalizer_registry is None:
- # This function may be called after this module's globals are
- # destroyed. See the _exit_function function in this module for more
- # notes.
- return
-
- if minpriority is None:
- f = lambda p : p[0] is not None
- else:
- f = lambda p : p[0] is not None and p[0] >= minpriority
-
- # Careful: _finalizer_registry may be mutated while this function
- # is running (either by a GC run or by another thread).
-
- # list(_finalizer_registry) should be atomic, while
- # list(_finalizer_registry.items()) is not.
- keys = [key for key in list(_finalizer_registry) if f(key)]
- keys.sort(reverse=True)
-
- for key in keys:
- finalizer = _finalizer_registry.get(key)
- # key may have been removed from the registry
- if finalizer is not None:
- sub_debug('calling %s', finalizer)
- try:
- finalizer()
- except Exception:
- import traceback
- traceback.print_exc()
-
- if minpriority is None:
- _finalizer_registry.clear()
-
-#
-# Clean up on exit
-#
-
-def is_exiting():
- '''
- Returns true if the process is shutting down
- '''
- return _exiting or _exiting is None
-
-_exiting = False
-
-def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
- active_children=process.active_children,
- current_process=process.current_process):
- # We hold on to references to functions in the arglist due to the
- # situation described below, where this function is called after this
- # module's globals are destroyed.
-
- global _exiting
-
- if not _exiting:
- _exiting = True
-
- info('process shutting down')
- debug('running all "atexit" finalizers with priority >= 0')
- _run_finalizers(0)
-
- if current_process() is not None:
- # We check if the current process is None here because if
- # it's None, any call to ``active_children()`` will raise
- # an AttributeError (active_children winds up trying to
- # get attributes from util._current_process). One
- # situation where this can happen is if someone has
- # manipulated sys.modules, causing this module to be
- # garbage collected. The destructor for the module type
- # then replaces all values in the module dict with None.
- # For instance, after setuptools runs a test it replaces
- # sys.modules with a copy created earlier. See issues
- # #9775 and #15881. Also related: #4106, #9205, and
- # #9207.
-
- for p in active_children():
- if p.daemon:
- info('calling terminate() for daemon %s', p.name)
- p._popen.terminate()
-
- for p in active_children():
- info('calling join() for process %s', p.name)
- p.join()
-
- debug('running the remaining "atexit" finalizers')
- _run_finalizers()
-
-atexit.register(_exit_function)
-
-#
-# Some fork aware types
-#
-
-class ForkAwareThreadLock(object):
- def __init__(self):
- self._lock = threading.Lock()
- self.acquire = self._lock.acquire
- self.release = self._lock.release
+ return x + '>'
+
+
+def _run_finalizers(minpriority=None):
+ '''
+ Run all finalizers whose exit priority is not None and at least minpriority
+
+ Finalizers with highest priority are called first; finalizers with
+ the same priority will be called in reverse order of creation.
+ '''
+ if _finalizer_registry is None:
+ # This function may be called after this module's globals are
+ # destroyed. See the _exit_function function in this module for more
+ # notes.
+ return
+
+ if minpriority is None:
+ f = lambda p : p[0] is not None
+ else:
+ f = lambda p : p[0] is not None and p[0] >= minpriority
+
+ # Careful: _finalizer_registry may be mutated while this function
+ # is running (either by a GC run or by another thread).
+
+ # list(_finalizer_registry) should be atomic, while
+ # list(_finalizer_registry.items()) is not.
+ keys = [key for key in list(_finalizer_registry) if f(key)]
+ keys.sort(reverse=True)
+
+ for key in keys:
+ finalizer = _finalizer_registry.get(key)
+ # key may have been removed from the registry
+ if finalizer is not None:
+ sub_debug('calling %s', finalizer)
+ try:
+ finalizer()
+ except Exception:
+ import traceback
+ traceback.print_exc()
+
+ if minpriority is None:
+ _finalizer_registry.clear()
+
+#
+# Clean up on exit
+#
+
+def is_exiting():
+ '''
+ Returns true if the process is shutting down
+ '''
+ return _exiting or _exiting is None
+
+_exiting = False
+
+def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
+ active_children=process.active_children,
+ current_process=process.current_process):
+ # We hold on to references to functions in the arglist due to the
+ # situation described below, where this function is called after this
+ # module's globals are destroyed.
+
+ global _exiting
+
+ if not _exiting:
+ _exiting = True
+
+ info('process shutting down')
+ debug('running all "atexit" finalizers with priority >= 0')
+ _run_finalizers(0)
+
+ if current_process() is not None:
+ # We check if the current process is None here because if
+ # it's None, any call to ``active_children()`` will raise
+ # an AttributeError (active_children winds up trying to
+ # get attributes from util._current_process). One
+ # situation where this can happen is if someone has
+ # manipulated sys.modules, causing this module to be
+ # garbage collected. The destructor for the module type
+ # then replaces all values in the module dict with None.
+ # For instance, after setuptools runs a test it replaces
+ # sys.modules with a copy created earlier. See issues
+ # #9775 and #15881. Also related: #4106, #9205, and
+ # #9207.
+
+ for p in active_children():
+ if p.daemon:
+ info('calling terminate() for daemon %s', p.name)
+ p._popen.terminate()
+
+ for p in active_children():
+ info('calling join() for process %s', p.name)
+ p.join()
+
+ debug('running the remaining "atexit" finalizers')
+ _run_finalizers()
+
+atexit.register(_exit_function)
+
+#
+# Some fork aware types
+#
+
+class ForkAwareThreadLock(object):
+ def __init__(self):
+ self._lock = threading.Lock()
+ self.acquire = self._lock.acquire
+ self.release = self._lock.release
register_after_fork(self, ForkAwareThreadLock._at_fork_reinit)
-
+
def _at_fork_reinit(self):
self._lock._at_fork_reinit()
- def __enter__(self):
- return self._lock.__enter__()
-
- def __exit__(self, *args):
- return self._lock.__exit__(*args)
-
-
-class ForkAwareLocal(threading.local):
- def __new__(cls):
- self = threading.local.__new__(cls)
- register_after_fork(self, lambda obj: obj.__dict__.clear())
- return self
-
- def __reduce__(self):
- return type(self), ()
-
-#
-# Close fds except those specified
-#
-
-try:
- MAXFD = os.sysconf("SC_OPEN_MAX")
-except Exception:
- MAXFD = 256
-
-def close_all_fds_except(fds):
- fds = list(fds) + [-1, MAXFD]
- fds.sort()
- assert fds[-1] == MAXFD, 'fd too large'
- for i in range(len(fds) - 1):
- os.closerange(fds[i]+1, fds[i+1])
-#
-# Close sys.stdin and replace stdin with os.devnull
-#
-
-def _close_stdin():
- if sys.stdin is None:
- return
-
- try:
- sys.stdin.close()
- except (OSError, ValueError):
- pass
-
- try:
- fd = os.open(os.devnull, os.O_RDONLY)
- try:
- sys.stdin = open(fd, closefd=False)
- except:
- os.close(fd)
- raise
- except (OSError, ValueError):
- pass
-
-#
-# Flush standard streams, if any
-#
-
-def _flush_std_streams():
- try:
- sys.stdout.flush()
- except (AttributeError, ValueError):
- pass
- try:
- sys.stderr.flush()
- except (AttributeError, ValueError):
- pass
-
-#
-# Start a program with only specified fds kept open
-#
-
-
-def _env_list():
- # Based on fork_exec in subprocess.py.
- env = os.environ.copy()
- env['Y_PYTHON_ENTRY_POINT'] = ':main'
- env_list = []
- for k, v in env.items():
- if '=' not in k:
- env_list.append(os.fsencode(k) + b'=' + os.fsencode(v))
- return env_list
-
-
-def spawnv_passfds(path, args, passfds):
- import _posixsubprocess
- passfds = tuple(sorted(map(int, passfds)))
- errpipe_read, errpipe_write = os.pipe()
- try:
- return _posixsubprocess.fork_exec(
- args, [os.fsencode(path)], True, passfds, None, _env_list(),
- -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
+ def __enter__(self):
+ return self._lock.__enter__()
+
+ def __exit__(self, *args):
+ return self._lock.__exit__(*args)
+
+
+class ForkAwareLocal(threading.local):
+ def __new__(cls):
+ self = threading.local.__new__(cls)
+ register_after_fork(self, lambda obj: obj.__dict__.clear())
+ return self
+
+ def __reduce__(self):
+ return type(self), ()
+
+#
+# Close fds except those specified
+#
+
+try:
+ MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+ MAXFD = 256
+
+def close_all_fds_except(fds):
+ fds = list(fds) + [-1, MAXFD]
+ fds.sort()
+ assert fds[-1] == MAXFD, 'fd too large'
+ for i in range(len(fds) - 1):
+ os.closerange(fds[i]+1, fds[i+1])
+#
+# Close sys.stdin and replace stdin with os.devnull
+#
+
+def _close_stdin():
+ if sys.stdin is None:
+ return
+
+ try:
+ sys.stdin.close()
+ except (OSError, ValueError):
+ pass
+
+ try:
+ fd = os.open(os.devnull, os.O_RDONLY)
+ try:
+ sys.stdin = open(fd, closefd=False)
+ except:
+ os.close(fd)
+ raise
+ except (OSError, ValueError):
+ pass
+
+#
+# Flush standard streams, if any
+#
+
+def _flush_std_streams():
+ try:
+ sys.stdout.flush()
+ except (AttributeError, ValueError):
+ pass
+ try:
+ sys.stderr.flush()
+ except (AttributeError, ValueError):
+ pass
+
+#
+# Start a program with only specified fds kept open
+#
+
+
+def _env_list():
+ # Based on fork_exec in subprocess.py.
+ env = os.environ.copy()
+ env['Y_PYTHON_ENTRY_POINT'] = ':main'
+ env_list = []
+ for k, v in env.items():
+ if '=' not in k:
+ env_list.append(os.fsencode(k) + b'=' + os.fsencode(v))
+ return env_list
+
+
+def spawnv_passfds(path, args, passfds):
+ import _posixsubprocess
+ passfds = tuple(sorted(map(int, passfds)))
+ errpipe_read, errpipe_write = os.pipe()
+ try:
+ return _posixsubprocess.fork_exec(
+ args, [os.fsencode(path)], True, passfds, None, _env_list(),
+ -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
False, False, None, None, None, -1, None)
- finally:
- os.close(errpipe_read)
- os.close(errpipe_write)
+ finally:
+ os.close(errpipe_read)
+ os.close(errpipe_write)
def close_fds(*fds):