diff options
author | alevitskii <[email protected]> | 2025-09-02 06:57:05 +0300 |
---|---|---|
committer | alevitskii <[email protected]> | 2025-09-02 07:19:27 +0300 |
commit | 7e399723cf6d967e20c8f9d2ee975426636242c5 (patch) | |
tree | abd5283daba11c07afc8fc16f02aec2c19e2272a /contrib/python/portalocker/py3 | |
parent | 14e9c865541d5abe545cb496c0143e4905b00c78 (diff) |
Drop LINTER param from styling macroses and cleanup deps
Drop LINTER param from styling macroses
commit_hash:00dd2e2ee103e509cff38f520d4779974abe39a7
Diffstat (limited to 'contrib/python/portalocker/py3')
24 files changed, 0 insertions, 2765 deletions
diff --git a/contrib/python/portalocker/py3/.dist-info/METADATA b/contrib/python/portalocker/py3/.dist-info/METADATA deleted file mode 100644 index 8202928049b..00000000000 --- a/contrib/python/portalocker/py3/.dist-info/METADATA +++ /dev/null @@ -1,259 +0,0 @@ -Metadata-Version: 2.4 -Name: portalocker -Version: 3.2.0 -Summary: Wraps the portalocker recipe for easy usage -Author-email: Rick van Hattem <[email protected]> -License-Expression: BSD-3-Clause -Project-URL: bugs, https://github.com/wolph/portalocker/issues -Project-URL: documentation, https://portalocker.readthedocs.io/en/latest/ -Project-URL: repository, https://github.com/wolph/portalocker/ -Keywords: locking,locks,with,statement,windows,linux,unix -Platform: any -Classifier: Development Status :: 5 - Production/Stable -Classifier: Development Status :: 6 - Mature -Classifier: Intended Audience :: Developers -Classifier: Natural Language :: English -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Operating System :: MacOS -Classifier: Operating System :: Microsoft :: MS-DOS -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: Microsoft -Classifier: Operating System :: POSIX :: BSD :: FreeBSD -Classifier: Operating System :: POSIX :: BSD -Classifier: Operating System :: POSIX :: Linux -Classifier: Operating System :: POSIX :: SunOS/Solaris -Classifier: Operating System :: POSIX -Classifier: Operating System :: Unix -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: IronPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Programming Language :: Python :: Implementation -Classifier: Programming Language :: Python -Classifier: Topic :: Education :: Testing -Classifier: Topic :: Office/Business -Classifier: Topic :: Other/Nonlisted Topic -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: System :: Monitoring -Classifier: Typing :: Typed -Requires-Python: >=3.9 -Description-Content-Type: text/x-rst -License-File: LICENSE -Requires-Dist: pywin32>=226; platform_system == "Windows" -Provides-Extra: docs -Requires-Dist: portalocker[tests]; extra == "docs" -Provides-Extra: tests -Requires-Dist: coverage-conditional-plugin>=0.9.0; extra == "tests" -Requires-Dist: portalocker[redis]; extra == "tests" -Requires-Dist: pytest-cov>=2.8.1; extra == "tests" -Requires-Dist: pytest-mypy>=0.8.0; extra == "tests" -Requires-Dist: pytest-rerunfailures>=15.0; extra == "tests" -Requires-Dist: pytest-timeout>=2.1.0; extra == "tests" -Requires-Dist: pytest>=5.4.1; extra == "tests" -Requires-Dist: sphinx>=6.0.0; extra == "tests" -Requires-Dist: types-pywin32>=310.0.0.20250429; extra == "tests" -Requires-Dist: types-redis; extra == "tests" -Provides-Extra: redis -Requires-Dist: redis; extra == "redis" -Dynamic: license-file - -############################################ -portalocker - Cross-platform locking library -############################################ - -.. image:: https://github.com/WoLpH/portalocker/actions/workflows/python-package.yml/badge.svg?branch=master - :alt: Linux Test Status - :target: https://github.com/WoLpH/portalocker/actions/ - -.. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true - :alt: Windows Tests Status - :target: https://ci.appveyor.com/project/WoLpH/portalocker - -.. image:: https://coveralls.io/repos/WoLpH/portalocker/badge.svg?branch=master - :alt: Coverage Status - :target: https://coveralls.io/r/WoLpH/portalocker?branch=master - -Overview --------- - -Portalocker is a library to provide an easy API to file locking. - -An important detail to note is that on Linux and Unix systems the locks are -advisory by default. By specifying the `-o mand` option to the mount command it -is possible to enable mandatory file locking on Linux. This is generally not -recommended however. For more information about the subject: - - - https://en.wikipedia.org/wiki/File_locking - - http://stackoverflow.com/questions/39292051/portalocker-does-not-seem-to-lock - - https://stackoverflow.com/questions/12062466/mandatory-file-lock-on-linux - -The module is currently maintained by Rick van Hattem <[email protected]>. -The project resides at https://github.com/WoLpH/portalocker . Bugs and feature -requests can be submitted there. Patches are also very welcome. - -Security contact information ------------------------------------------------------------------------------- - -To report a security vulnerability, please use the -`Tidelift security contact <https://tidelift.com/security>`_. -Tidelift will coordinate the fix and disclosure. - -Redis Locks ------------ - -This library now features a lock based on Redis which allows for locks across -multiple threads, processes and even distributed locks across multiple -computers. - -It is an extremely reliable Redis lock that is based on pubsub. - -As opposed to most Redis locking systems based on key/value pairs, -this locking method is based on the pubsub system. The big advantage is -that if the connection gets killed due to network issues, crashing -processes or otherwise, it will still immediately unlock instead of -waiting for a lock timeout. - -First make sure you have everything installed correctly: - -:: - - pip install "portalocker[redis]" - -Usage is really easy: - -:: - - import portalocker - - lock = portalocker.RedisLock('some_lock_channel_name') - - with lock: - print('do something here') - -The API is essentially identical to the other ``Lock`` classes so in addition -to the ``with`` statement you can also use ``lock.acquire(...)``. - -Python 2 --------- - -Python 2 was supported in versions before Portalocker 2.0. If you are still -using -Python 2, -you can run this to install: - -:: - - pip install "portalocker<2" - -Tips ----- - -On some networked filesystems it might be needed to force a `os.fsync()` before -closing the file so it's actually written before another client reads the file. -Effectively this comes down to: - -:: - - with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: - # do what you need to do - ... - - # flush and sync to filesystem - fh.flush() - os.fsync(fh.fileno()) - -Links ------ - -* Documentation - - http://portalocker.readthedocs.org/en/latest/ -* Source - - https://github.com/WoLpH/portalocker -* Bug reports - - https://github.com/WoLpH/portalocker/issues -* Package homepage - - https://pypi.python.org/pypi/portalocker -* My blog - - http://w.wol.ph/ - -Examples --------- - -To make sure your cache generation scripts don't race, use the `Lock` class: - ->>> import portalocker ->>> with portalocker.Lock('somefile', timeout=1) as fh: -... print('writing some stuff to my cache...', file=fh) - -To customize the opening and locking a manual approach is also possible: - ->>> import portalocker ->>> file = open('somefile', 'r+') ->>> portalocker.lock(file, portalocker.LockFlags.EXCLUSIVE) ->>> file.seek(12) ->>> file.write('foo') ->>> file.close() - -Explicitly unlocking is not needed in most cases but omitting it has been known -to cause issues: -https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 - -If needed, it can be done through: - ->>> portalocker.unlock(file) - -Do note that your data might still be in a buffer so it is possible that your -data is not available until you `flush()` or `close()`. - -To create a cross platform bounded semaphore across multiple processes you can -use the `BoundedSemaphore` class which functions somewhat similar to -`threading.BoundedSemaphore`: - ->>> import portalocker ->>> n = 2 ->>> timeout = 0.1 - ->>> semaphore_a = portalocker.BoundedSemaphore(n, timeout=timeout) ->>> semaphore_b = portalocker.BoundedSemaphore(n, timeout=timeout) ->>> semaphore_c = portalocker.BoundedSemaphore(n, timeout=timeout) - ->>> semaphore_a.acquire() -<portalocker.utils.Lock object at ...> ->>> semaphore_b.acquire() -<portalocker.utils.Lock object at ...> ->>> semaphore_c.acquire() -Traceback (most recent call last): - ... -portalocker.exceptions.AlreadyLocked - - -More examples can be found in the -`tests <http://portalocker.readthedocs.io/en/latest/_modules/tests/tests.html>`_. - - -Versioning ----------- - -This library follows `Semantic Versioning <http://semver.org/>`_. - - -Changelog ---------- - -Every release has a ``git tag`` with a commit message for the tag -explaining what was added and/or changed. The list of tags/releases -including the commit messages can be found here: -https://github.com/WoLpH/portalocker/releases - -License -------- - -See the `LICENSE <https://github.com/WoLpH/portalocker/blob/develop/LICENSE>`_ file. - diff --git a/contrib/python/portalocker/py3/.dist-info/top_level.txt b/contrib/python/portalocker/py3/.dist-info/top_level.txt deleted file mode 100644 index 7bbc14e6fa6..00000000000 --- a/contrib/python/portalocker/py3/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -portalocker diff --git a/contrib/python/portalocker/py3/.yandex_meta/yamaker.yaml b/contrib/python/portalocker/py3/.yandex_meta/yamaker.yaml deleted file mode 100644 index bca0c4e5337..00000000000 --- a/contrib/python/portalocker/py3/.yandex_meta/yamaker.yaml +++ /dev/null @@ -1,4 +0,0 @@ -keep: - - tests/* -exclude: - - portalocker_tests/* diff --git a/contrib/python/portalocker/py3/LICENSE b/contrib/python/portalocker/py3/LICENSE deleted file mode 100644 index b638bda0d3b..00000000000 --- a/contrib/python/portalocker/py3/LICENSE +++ /dev/null @@ -1,11 +0,0 @@ -Copyright 2022 Rick van Hattem - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/portalocker/py3/README.rst b/contrib/python/portalocker/py3/README.rst deleted file mode 100644 index c5ef42f614b..00000000000 --- a/contrib/python/portalocker/py3/README.rst +++ /dev/null @@ -1,193 +0,0 @@ -############################################ -portalocker - Cross-platform locking library -############################################ - -.. image:: https://github.com/WoLpH/portalocker/actions/workflows/python-package.yml/badge.svg?branch=master - :alt: Linux Test Status - :target: https://github.com/WoLpH/portalocker/actions/ - -.. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true - :alt: Windows Tests Status - :target: https://ci.appveyor.com/project/WoLpH/portalocker - -.. image:: https://coveralls.io/repos/WoLpH/portalocker/badge.svg?branch=master - :alt: Coverage Status - :target: https://coveralls.io/r/WoLpH/portalocker?branch=master - -Overview --------- - -Portalocker is a library to provide an easy API to file locking. - -An important detail to note is that on Linux and Unix systems the locks are -advisory by default. By specifying the `-o mand` option to the mount command it -is possible to enable mandatory file locking on Linux. This is generally not -recommended however. For more information about the subject: - - - https://en.wikipedia.org/wiki/File_locking - - http://stackoverflow.com/questions/39292051/portalocker-does-not-seem-to-lock - - https://stackoverflow.com/questions/12062466/mandatory-file-lock-on-linux - -The module is currently maintained by Rick van Hattem <[email protected]>. -The project resides at https://github.com/WoLpH/portalocker . Bugs and feature -requests can be submitted there. Patches are also very welcome. - -Security contact information ------------------------------------------------------------------------------- - -To report a security vulnerability, please use the -`Tidelift security contact <https://tidelift.com/security>`_. -Tidelift will coordinate the fix and disclosure. - -Redis Locks ------------ - -This library now features a lock based on Redis which allows for locks across -multiple threads, processes and even distributed locks across multiple -computers. - -It is an extremely reliable Redis lock that is based on pubsub. - -As opposed to most Redis locking systems based on key/value pairs, -this locking method is based on the pubsub system. The big advantage is -that if the connection gets killed due to network issues, crashing -processes or otherwise, it will still immediately unlock instead of -waiting for a lock timeout. - -First make sure you have everything installed correctly: - -:: - - pip install "portalocker[redis]" - -Usage is really easy: - -:: - - import portalocker - - lock = portalocker.RedisLock('some_lock_channel_name') - - with lock: - print('do something here') - -The API is essentially identical to the other ``Lock`` classes so in addition -to the ``with`` statement you can also use ``lock.acquire(...)``. - -Python 2 --------- - -Python 2 was supported in versions before Portalocker 2.0. If you are still -using -Python 2, -you can run this to install: - -:: - - pip install "portalocker<2" - -Tips ----- - -On some networked filesystems it might be needed to force a `os.fsync()` before -closing the file so it's actually written before another client reads the file. -Effectively this comes down to: - -:: - - with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: - # do what you need to do - ... - - # flush and sync to filesystem - fh.flush() - os.fsync(fh.fileno()) - -Links ------ - -* Documentation - - http://portalocker.readthedocs.org/en/latest/ -* Source - - https://github.com/WoLpH/portalocker -* Bug reports - - https://github.com/WoLpH/portalocker/issues -* Package homepage - - https://pypi.python.org/pypi/portalocker -* My blog - - http://w.wol.ph/ - -Examples --------- - -To make sure your cache generation scripts don't race, use the `Lock` class: - ->>> import portalocker ->>> with portalocker.Lock('somefile', timeout=1) as fh: -... print('writing some stuff to my cache...', file=fh) - -To customize the opening and locking a manual approach is also possible: - ->>> import portalocker ->>> file = open('somefile', 'r+') ->>> portalocker.lock(file, portalocker.LockFlags.EXCLUSIVE) ->>> file.seek(12) ->>> file.write('foo') ->>> file.close() - -Explicitly unlocking is not needed in most cases but omitting it has been known -to cause issues: -https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 - -If needed, it can be done through: - ->>> portalocker.unlock(file) - -Do note that your data might still be in a buffer so it is possible that your -data is not available until you `flush()` or `close()`. - -To create a cross platform bounded semaphore across multiple processes you can -use the `BoundedSemaphore` class which functions somewhat similar to -`threading.BoundedSemaphore`: - ->>> import portalocker ->>> n = 2 ->>> timeout = 0.1 - ->>> semaphore_a = portalocker.BoundedSemaphore(n, timeout=timeout) ->>> semaphore_b = portalocker.BoundedSemaphore(n, timeout=timeout) ->>> semaphore_c = portalocker.BoundedSemaphore(n, timeout=timeout) - ->>> semaphore_a.acquire() -<portalocker.utils.Lock object at ...> ->>> semaphore_b.acquire() -<portalocker.utils.Lock object at ...> ->>> semaphore_c.acquire() -Traceback (most recent call last): - ... -portalocker.exceptions.AlreadyLocked - - -More examples can be found in the -`tests <http://portalocker.readthedocs.io/en/latest/_modules/tests/tests.html>`_. - - -Versioning ----------- - -This library follows `Semantic Versioning <http://semver.org/>`_. - - -Changelog ---------- - -Every release has a ``git tag`` with a commit message for the tag -explaining what was added and/or changed. The list of tags/releases -including the commit messages can be found here: -https://github.com/WoLpH/portalocker/releases - -License -------- - -See the `LICENSE <https://github.com/WoLpH/portalocker/blob/develop/LICENSE>`_ file. - diff --git a/contrib/python/portalocker/py3/portalocker/__about__.py b/contrib/python/portalocker/py3/portalocker/__about__.py deleted file mode 100644 index 10a10202531..00000000000 --- a/contrib/python/portalocker/py3/portalocker/__about__.py +++ /dev/null @@ -1,6 +0,0 @@ -__package_name__ = 'portalocker' -__author__ = 'Rick van Hattem' -__email__ = '[email protected]' -__version__ = '3.2.0' -__description__ = """Wraps the portalocker recipe for easy usage""" -__url__ = 'https://github.com/WoLpH/portalocker' diff --git a/contrib/python/portalocker/py3/portalocker/__init__.py b/contrib/python/portalocker/py3/portalocker/__init__.py deleted file mode 100644 index b1c451d2a51..00000000000 --- a/contrib/python/portalocker/py3/portalocker/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -from . import __about__, constants, exceptions, portalocker -from .utils import ( - BoundedSemaphore, - Lock, - RLock, - TemporaryFileLock, - open_atomic, -) - -try: # pragma: no cover - from .redis import RedisLock -except ImportError: # pragma: no cover - RedisLock = None # type: ignore[assignment,misc] - - -#: The package name on Pypi -__package_name__ = __about__.__package_name__ -#: Current author and maintainer, view the git history for the previous ones -__author__ = __about__.__author__ -#: Current author's email address -__email__ = __about__.__email__ -#: Version number -__version__ = '3.2.0' -#: Package description for Pypi -__description__ = __about__.__description__ -#: Package homepage -__url__ = __about__.__url__ - - -#: Exception thrown when the file is already locked by someone else -AlreadyLocked = exceptions.AlreadyLocked -#: Exception thrown if an error occurred during locking -LockException = exceptions.LockException - - -#: Lock a file. Note that this is an advisory lock on Linux/Unix systems -lock = portalocker.lock -#: Unlock a file -unlock = portalocker.unlock - -#: Place an exclusive lock. -#: Only one process may hold an exclusive lock for a given file at a given -#: time. -LOCK_EX: constants.LockFlags = constants.LockFlags.EXCLUSIVE - -#: Place a shared lock. -#: More than one process may hold a shared lock for a given file at a given -#: time. -LOCK_SH: constants.LockFlags = constants.LockFlags.SHARED - -#: Acquire the lock in a non-blocking fashion. -LOCK_NB: constants.LockFlags = constants.LockFlags.NON_BLOCKING - -#: Remove an existing lock held by this process. -LOCK_UN: constants.LockFlags = constants.LockFlags.UNBLOCK - -#: Locking flags enum -LockFlags = constants.LockFlags - -#: Locking utility class to automatically handle opening with timeouts and -#: context wrappers - -__all__ = [ - 'LOCK_EX', - 'LOCK_NB', - 'LOCK_SH', - 'LOCK_UN', - 'AlreadyLocked', - 'BoundedSemaphore', - 'Lock', - 'LockException', - 'LockFlags', - 'RLock', - 'RedisLock', - 'TemporaryFileLock', - 'lock', - 'open_atomic', - 'unlock', -] diff --git a/contrib/python/portalocker/py3/portalocker/__main__.py b/contrib/python/portalocker/py3/portalocker/__main__.py deleted file mode 100644 index a573ad7931e..00000000000 --- a/contrib/python/portalocker/py3/portalocker/__main__.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -import argparse -import logging -import pathlib -import re -import subprocess -import typing - -base_path = pathlib.Path(__file__).parent.parent -src_path = base_path / 'portalocker' -dist_path = base_path / 'dist' -_default_output_path = base_path / 'dist' / 'portalocker.py' - -_NAMES_RE = re.compile(r'(?P<names>[^()]+)$') -_RELATIVE_IMPORT_RE = re.compile( - r'^from \.(?P<from>.*?) import (?P<paren>\(?)(?P<names>[^()]+)$', -) -_USELESS_ASSIGNMENT_RE = re.compile(r'^(?P<name>\w+) = \1\n$') - -_TEXT_TEMPLATE = """''' -{} -''' - -""" - -logger = logging.getLogger(__name__) - - -def main(argv: typing.Sequence[str] | None = None) -> None: - parser = argparse.ArgumentParser() - - subparsers = parser.add_subparsers(required=True) - combine_parser = subparsers.add_parser( - 'combine', - help='Combine all Python files into a single unified `portalocker.py` ' - 'file for easy distribution', - ) - combine_parser.add_argument( - '--output-file', - '-o', - type=argparse.FileType('w'), - default=str(_default_output_path), - ) - - combine_parser.set_defaults(func=combine) - args = parser.parse_args(argv) - args.func(args) - - -def _read_file( # noqa: C901 - path: pathlib.Path, - seen_files: set[pathlib.Path], -) -> typing.Iterator[str]: - if path in seen_files: - return - - names: set[str] = set() - seen_files.add(path) - paren = False - from_ = None - try: - for line in path.open(encoding='ascii'): - if '__future__' in line: - continue - - if paren: - if ')' in line: - line = line.split(')', 1)[1] - paren = False - continue - - match = _NAMES_RE.match(line) - else: - match = _RELATIVE_IMPORT_RE.match(line) - - if match: - if not paren: - paren = bool(match.group('paren')) - from_ = match.group('from') - - if from_: - names.add(from_) - yield from _read_file(src_path / f'{from_}.py', seen_files) - else: - for name in match.group('names').split(','): - name = name.strip() - names.add(name) - yield from _read_file( - src_path / f'{name}.py', seen_files - ) - else: - yield _clean_line(line, names) - except UnicodeDecodeError as exception: # pragma: no cover - _, text, start_byte, end_byte, error = exception.args - - offset = 100 - snippet = text[start_byte - offset : end_byte + offset] - logger.error( # noqa: TRY400 - f'Invalid encoding for {path}: {error} at byte ' - f'({start_byte}:{end_byte})\n' - f'Snippet: {snippet!r}' - ) - raise - - -def _clean_line(line: str, names: set[str]) -> str: - # Replace `some_import.spam` with `spam` - if names: - joined_names = '|'.join(names) - line = re.sub(rf'\b({joined_names})\.', '', line) - - # Replace useless assignments (e.g. `spam = spam`) - return _USELESS_ASSIGNMENT_RE.sub('', line) - - -def combine(args: argparse.Namespace) -> None: - output_file = args.output_file - pathlib.Path(output_file.name).parent.mkdir(parents=True, exist_ok=True) - - # We're handling this separately because it has to be the first import. - output_file.write('from __future__ import annotations\n') - - output_file.write( - _TEXT_TEMPLATE.format( - (base_path / 'README.rst').read_text(encoding='ascii') - ), - ) - output_file.write( - _TEXT_TEMPLATE.format( - (base_path / 'LICENSE').read_text(encoding='ascii') - ), - ) - - seen_files: set[pathlib.Path] = set() - for line in _read_file(src_path / '__init__.py', seen_files): - output_file.write(line) - - output_file.flush() - output_file.close() - - logger.info(f'Wrote combined file to {output_file.name}') - # Run ruff if available. If not then just run the file. - try: # pragma: no cover - subprocess.run(['ruff', 'format', output_file.name], timeout=3) - subprocess.run( - ['ruff', 'check', '--fix', '--fix-only', output_file.name], - timeout=3, - ) - except FileNotFoundError: # pragma: no cover - logger.warning( - 'Ruff is not installed. Skipping linting and formatting step.' - ) - subprocess.run(['python3', output_file.name]) - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - main() diff --git a/contrib/python/portalocker/py3/portalocker/constants.py b/contrib/python/portalocker/py3/portalocker/constants.py deleted file mode 100644 index 5787725e727..00000000000 --- a/contrib/python/portalocker/py3/portalocker/constants.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Locking constants - -Lock types: - -- `EXCLUSIVE` exclusive lock -- `SHARED` shared lock - -Lock flags: - -- `NON_BLOCKING` non-blocking - -Manually unlock, only needed internally - -- `UNBLOCK` unlock -""" - -import enum -import os - -# The actual tests will execute the code anyhow so the following code can -# safely be ignored from the coverage tests -if os.name == 'nt': # pragma: no cover - import msvcrt - - #: exclusive lock - LOCK_EX = 0x1 - #: shared lock - LOCK_SH = 0x2 - #: non-blocking - LOCK_NB = 0x4 - #: unlock - LOCK_UN = msvcrt.LK_UNLCK # type: ignore[attr-defined] - -elif os.name == 'posix': # pragma: no cover - import fcntl - - #: exclusive lock - LOCK_EX = fcntl.LOCK_EX # type: ignore[attr-defined] - #: shared lock - LOCK_SH = fcntl.LOCK_SH # type: ignore[attr-defined] - #: non-blocking - LOCK_NB = fcntl.LOCK_NB # type: ignore[attr-defined] - #: unlock - LOCK_UN = fcntl.LOCK_UN # type: ignore[attr-defined] - -else: # pragma: no cover - raise RuntimeError('PortaLocker only defined for nt and posix platforms') - - -class LockFlags(enum.IntFlag): - #: exclusive lock - EXCLUSIVE = LOCK_EX - #: shared lock - SHARED = LOCK_SH - #: non-blocking - NON_BLOCKING = LOCK_NB - #: unlock - UNBLOCK = LOCK_UN diff --git a/contrib/python/portalocker/py3/portalocker/exceptions.py b/contrib/python/portalocker/py3/portalocker/exceptions.py deleted file mode 100644 index 00142b0e811..00000000000 --- a/contrib/python/portalocker/py3/portalocker/exceptions.py +++ /dev/null @@ -1,36 +0,0 @@ -import typing - -from . import types - - -class BaseLockException(Exception): # noqa: N818 - # Error codes: - LOCK_FAILED: typing.Final = 1 - - strerror: typing.Optional[str] = None # ensure attribute always exists - - def __init__( - self, - *args: typing.Any, - fh: typing.Union[types.IO, None, int, types.HasFileno] = None, - **kwargs: typing.Any, - ) -> None: - self.fh = fh - self.strerror = ( - str(args[1]) - if len(args) > 1 and isinstance(args[1], str) - else None - ) - Exception.__init__(self, *args) - - -class LockException(BaseLockException): - pass - - -class AlreadyLocked(LockException): - pass - - -class FileToLarge(LockException): - pass diff --git a/contrib/python/portalocker/py3/portalocker/portalocker.py b/contrib/python/portalocker/py3/portalocker/portalocker.py deleted file mode 100644 index 006cb57571e..00000000000 --- a/contrib/python/portalocker/py3/portalocker/portalocker.py +++ /dev/null @@ -1,444 +0,0 @@ -# pyright: reportUnknownMemberType=false, reportAttributeAccessIssue=false -"""Module portalocker. - -This module provides cross-platform file locking functionality. -The Windows implementation now supports two variants: - - 1. A default method using the Win32 API (win32file.LockFileEx/UnlockFileEx). - 2. An alternative that uses msvcrt.locking for exclusive locks (shared - locks still use the Win32 API). - -This version uses classes to encapsulate locking logic, while maintaining -the original external API, including the LOCKER constant for specific -backwards compatibility (POSIX) and Windows behavior. -""" - -import io -import os -import typing -from typing import ( - Any, - Callable, - Optional, - Union, - cast, -) - -from . import constants, exceptions, types - -# Alias for readability -LockFlags = constants.LockFlags - - -# Define a protocol for callable lockers -class LockCallable(typing.Protocol): - def __call__( - self, file_obj: types.FileArgument, flags: LockFlags - ) -> None: ... - - -class UnlockCallable(typing.Protocol): - def __call__(self, file_obj: types.FileArgument) -> None: ... - - -class BaseLocker: - """Base class for locker implementations.""" - - def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None: - """Lock the file.""" - raise NotImplementedError - - def unlock(self, file_obj: types.FileArgument) -> None: - """Unlock the file.""" - raise NotImplementedError - - -# Define refined LockerType with more specific types -LockerType = Union[ - # POSIX-style fcntl.flock callable - Callable[[Union[int, types.HasFileno], int], Any], - # Tuple of lock and unlock functions - tuple[LockCallable, UnlockCallable], - # BaseLocker instance - BaseLocker, - # BaseLocker class - type[BaseLocker], -] - -LOCKER: LockerType - -if os.name == 'nt': # pragma: not-posix - # Windows-specific helper functions - def _prepare_windows_file( - file_obj: types.FileArgument, - ) -> tuple[int, Optional[typing.IO[Any]], Optional[int]]: - """Prepare file for Windows: get fd, optionally seek and save pos.""" - if isinstance(file_obj, int): - # Plain file descriptor - return file_obj, None, None - - # Full IO objects (have tell/seek) -> preserve and restore position - if isinstance(file_obj, io.IOBase): - fd: int = file_obj.fileno() - original_pos = file_obj.tell() - if original_pos != 0: - file_obj.seek(0) - return fd, typing.cast(typing.IO[Any], file_obj), original_pos - # cast satisfies mypy: IOBase -> IO[Any] - - # Fallback: an object that only implements fileno() (HasFileno) - fd = typing.cast(types.HasFileno, file_obj).fileno() # type: ignore[redundant-cast] - return fd, None, None - - def _restore_windows_file_pos( - file_io_obj: Optional[typing.IO[Any]], - original_pos: Optional[int], - ) -> None: - """Restore file position if it was an IO object and pos was saved.""" - if file_io_obj and original_pos is not None and original_pos != 0: - file_io_obj.seek(original_pos) - - class Win32Locker(BaseLocker): - """Locker using Win32 API (LockFileEx/UnlockFileEx).""" - - _overlapped: Any # pywintypes.OVERLAPPED - _lock_bytes_low: int = -0x10000 - - def __init__(self) -> None: - try: - import pywintypes - except ImportError as e: - raise ImportError( - 'pywintypes is required for Win32Locker but not ' - 'found. Please install pywin32.' - ) from e - self._overlapped = pywintypes.OVERLAPPED() - - def _get_os_handle(self, fd: int) -> int: - try: - import msvcrt - except ImportError as e: - raise ImportError( - 'msvcrt is required for _get_os_handle on Windows ' - 'but not found.' - ) from e - return cast(int, msvcrt.get_osfhandle(fd)) # type: ignore[attr-defined,redundant-cast] - - def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None: - import pywintypes - import win32con - import win32file - import winerror - - fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) - os_fh = self._get_os_handle(fd) - - mode = 0 - if flags & LockFlags.NON_BLOCKING: - mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY - if flags & LockFlags.EXCLUSIVE: - mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK - - try: - win32file.LockFileEx( - os_fh, mode, 0, self._lock_bytes_low, self._overlapped - ) - except pywintypes.error as exc_value: # type: ignore[misc] - if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: - raise exceptions.AlreadyLocked( - exceptions.LockException.LOCK_FAILED, - exc_value.strerror, - fh=file_obj, # Pass original file_obj - ) from exc_value - else: - raise - finally: - _restore_windows_file_pos(io_obj_ctx, pos_ctx) - - def unlock(self, file_obj: types.FileArgument) -> None: - import pywintypes - import win32file - import winerror - - fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) - os_fh = self._get_os_handle(fd) - - try: - win32file.UnlockFileEx( - os_fh, 0, self._lock_bytes_low, self._overlapped - ) - except pywintypes.error as exc: # type: ignore[misc] - if exc.winerror != winerror.ERROR_NOT_LOCKED: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - exc.strerror, - fh=file_obj, # Pass original file_obj - ) from exc - except OSError as exc: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - exc.strerror, - fh=file_obj, # Pass original file_obj - ) from exc - finally: - _restore_windows_file_pos(io_obj_ctx, pos_ctx) - - class MsvcrtLocker(BaseLocker): - _win32_locker: Win32Locker - _msvcrt_lock_length: int = 0x10000 - - def __init__(self) -> None: - self._win32_locker = Win32Locker() - try: - import msvcrt - except ImportError as e: - raise ImportError( - 'msvcrt is required for MsvcrtLocker but not found.' - ) from e - - attrs = ['LK_LOCK', 'LK_RLCK', 'LK_NBLCK', 'LK_UNLCK', 'LK_NBRLCK'] - defaults = [0, 1, 2, 3, 2] # LK_NBRLCK often same as LK_NBLCK (2) - for attr, default_val in zip(attrs, defaults): - if not hasattr(msvcrt, attr): - setattr(msvcrt, attr, default_val) - - def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None: - import msvcrt - - if flags & LockFlags.SHARED: - win32_api_flags = LockFlags(0) - if flags & LockFlags.NON_BLOCKING: - win32_api_flags |= LockFlags.NON_BLOCKING - self._win32_locker.lock(file_obj, win32_api_flags) - return - - fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) - mode = ( - msvcrt.LK_NBLCK # type: ignore[attr-defined] - if flags & LockFlags.NON_BLOCKING - else msvcrt.LK_LOCK # type: ignore[attr-defined] - ) - - try: - msvcrt.locking( # type: ignore[attr-defined] - fd, - mode, - self._msvcrt_lock_length, - ) - except OSError as exc_value: - if exc_value.errno in (13, 16, 33, 36): - raise exceptions.AlreadyLocked( - exceptions.LockException.LOCK_FAILED, - str(exc_value), - fh=file_obj, # Pass original file_obj - ) from exc_value - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - str(exc_value), - fh=file_obj, # Pass original file_obj - ) from exc_value - finally: - _restore_windows_file_pos(io_obj_ctx, pos_ctx) - - def unlock(self, file_obj: types.FileArgument) -> None: - import msvcrt - - fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) - took_fallback_path = False - - try: - msvcrt.locking( # type: ignore[attr-defined] - fd, - msvcrt.LK_UNLCK, # type: ignore[attr-defined] - self._msvcrt_lock_length, - ) - except OSError as exc: - if exc.errno == 13: # EACCES (Permission denied) - took_fallback_path = True - # Restore position before calling win32_locker, - # as it will re-prepare. - _restore_windows_file_pos(io_obj_ctx, pos_ctx) - try: - self._win32_locker.unlock( - file_obj - ) # win32_locker handles its own seeking - except exceptions.LockException as win32_exc: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - f'msvcrt unlock failed ({exc.strerror}), and ' - f'win32 fallback failed ({win32_exc.strerror})', - fh=file_obj, - ) from win32_exc - except Exception as final_exc: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - f'msvcrt unlock failed ({exc.strerror}), and ' - f'win32 fallback failed with unexpected error: ' - f'{final_exc!s}', - fh=file_obj, - ) from final_exc - else: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - exc.strerror, - fh=file_obj, - ) from exc - finally: - if not took_fallback_path: - _restore_windows_file_pos(io_obj_ctx, pos_ctx) - - _locker_instances: dict[type[BaseLocker], BaseLocker] = dict() - - LOCKER = MsvcrtLocker # type: ignore[reportConstantRedefinition] - - def lock(file: types.FileArgument, flags: LockFlags) -> None: - if isinstance(LOCKER, BaseLocker): - # If LOCKER is a BaseLocker instance, use its lock method - locker: Callable[[types.FileArgument, LockFlags], None] = ( - LOCKER.lock - ) - elif isinstance(LOCKER, tuple): - locker = LOCKER[0] # type: ignore[reportUnknownVariableType] - elif issubclass(LOCKER, BaseLocker): # type: ignore[unreachable,arg-type] # pyright: ignore [reportUnnecessaryIsInstance] - locker_instance = _locker_instances.get(LOCKER) # type: ignore[arg-type] - if locker_instance is None: - # Create an instance of the locker class if not already done - _locker_instances[LOCKER] = locker_instance = LOCKER() # type: ignore[ignore,index,call-arg] - - locker = locker_instance.lock - else: - raise TypeError( - f'LOCKER must be a BaseLocker instance, a tuple of lock and ' - f'unlock functions, or a subclass of BaseLocker, ' - f'got {type(LOCKER)}.' - ) - - locker(file, flags) - - def unlock(file: types.FileArgument) -> None: - if isinstance(LOCKER, BaseLocker): - # If LOCKER is a BaseLocker instance, use its lock method - unlocker: Callable[[types.FileArgument], None] = LOCKER.unlock - elif isinstance(LOCKER, tuple): - unlocker = LOCKER[1] # type: ignore[reportUnknownVariableType] - - elif issubclass(LOCKER, BaseLocker): # type: ignore[unreachable,arg-type] # pyright: ignore [reportUnnecessaryIsInstance] - locker_instance = _locker_instances.get(LOCKER) # type: ignore[arg-type] - if locker_instance is None: - # Create an instance of the locker class if not already done - _locker_instances[LOCKER] = locker_instance = LOCKER() # type: ignore[ignore,index,call-arg] - - unlocker = locker_instance.unlock - else: - raise TypeError( - f'LOCKER must be a BaseLocker instance, a tuple of lock and ' - f'unlock functions, or a subclass of BaseLocker, ' - f'got {type(LOCKER)}.' - ) - - unlocker(file) - -else: # pragma: not-nt - import errno - import fcntl - - # PosixLocker methods accept FileArgument | HasFileno - PosixFileArgument = Union[types.FileArgument, types.HasFileno] - - class PosixLocker(BaseLocker): - """Locker implementation using the `LOCKER` constant""" - - _locker: Optional[ - Callable[[Union[int, types.HasFileno], int], Any] - ] = None - - @property - def locker(self) -> Callable[[Union[int, types.HasFileno], int], Any]: - if self._locker is None: - # On POSIX systems ``LOCKER`` is a callable (fcntl.flock) but - # mypy also sees the Windows-only tuple assignment. Explicitly - # cast so mypy knows we are returning the callable variant - # here. - return cast( - Callable[[Union[int, types.HasFileno], int], Any], LOCKER - ) # pyright: ignore[reportUnnecessaryCast] - - # mypy does not realise ``self._locker`` is non-None after the - # check - assert self._locker is not None - return self._locker - - def _get_fd(self, file_obj: PosixFileArgument) -> int: - if isinstance(file_obj, int): - return file_obj - # Check for fileno() method; covers typing.IO and HasFileno - elif hasattr(file_obj, 'fileno') and callable(file_obj.fileno): - return file_obj.fileno() - else: - # Should not be reached if PosixFileArgument is correct. - # isinstance(file_obj, io.IOBase) could be an - # alternative check - # but hasattr is more general for HasFileno. - raise TypeError( - "Argument 'file_obj' must be an int, an IO object " - 'with fileno(), or implement HasFileno.' - ) - - def lock(self, file_obj: PosixFileArgument, flags: LockFlags) -> None: - if (flags & LockFlags.NON_BLOCKING) and not flags & ( - LockFlags.SHARED | LockFlags.EXCLUSIVE - ): - raise RuntimeError( - 'When locking in non-blocking mode on POSIX, ' - 'the SHARED or EXCLUSIVE flag must be specified as well.' - ) - - fd = self._get_fd(file_obj) - try: - self.locker(fd, flags) - except OSError as exc_value: - if exc_value.errno in (errno.EACCES, errno.EAGAIN): - raise exceptions.AlreadyLocked( - exc_value, - strerror=str(exc_value), - fh=file_obj, # Pass original file_obj - ) from exc_value - else: - raise exceptions.LockException( - exc_value, - strerror=str(exc_value), - fh=file_obj, # Pass original file_obj - ) from exc_value - except EOFError as exc_value: # NFS specific - raise exceptions.LockException( - exc_value, - strerror=str(exc_value), - fh=file_obj, # Pass original file_obj - ) from exc_value - - def unlock(self, file_obj: PosixFileArgument) -> None: - fd = self._get_fd(file_obj) - self.locker(fd, LockFlags.UNBLOCK) - - class FlockLocker(PosixLocker): - """FlockLocker is a PosixLocker implementation using fcntl.flock.""" - - LOCKER = fcntl.flock # type: ignore[attr-defined] - - class LockfLocker(PosixLocker): - """LockfLocker is a PosixLocker implementation using fcntl.lockf.""" - - LOCKER = fcntl.lockf # type: ignore[attr-defined] - - # LOCKER constant for POSIX is fcntl.flock for backward compatibility. - # Type matches: Callable[[Union[int, HasFileno], int], Any] - LOCKER = fcntl.flock # type: ignore[attr-defined,reportConstantRedefinition] - - _posix_locker_instance = PosixLocker() - - # Public API for POSIX uses the PosixLocker instance - def lock(file: types.FileArgument, flags: LockFlags) -> None: - _posix_locker_instance.lock(file, flags) - - def unlock(file: types.FileArgument) -> None: - _posix_locker_instance.unlock(file) diff --git a/contrib/python/portalocker/py3/portalocker/py.typed b/contrib/python/portalocker/py3/portalocker/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/portalocker/py3/portalocker/py.typed +++ /dev/null diff --git a/contrib/python/portalocker/py3/portalocker/redis.py b/contrib/python/portalocker/py3/portalocker/redis.py deleted file mode 100644 index f2e11e6b10b..00000000000 --- a/contrib/python/portalocker/py3/portalocker/redis.py +++ /dev/null @@ -1,270 +0,0 @@ -# pyright: reportUnknownMemberType=false -from __future__ import annotations - -import _thread -import json -import logging -import random -import time -import typing - -import redis - -from . import exceptions, utils - -logger = logging.getLogger(__name__) - -DEFAULT_UNAVAILABLE_TIMEOUT = 1 -DEFAULT_THREAD_SLEEP_TIME = 0.1 - - -class PubSubWorkerThread(redis.client.PubSubWorkerThread): - def run(self) -> None: - try: - super().run() - except Exception: # pragma: no cover - _thread.interrupt_main() - raise - - -class RedisLock(utils.LockBase): - """ - An extremely reliable Redis lock based on pubsub with a keep-alive thread - - As opposed to most Redis locking systems based on key/value pairs, - this locking method is based on the pubsub system. The big advantage is - that if the connection gets killed due to network issues, crashing - processes or otherwise, it will still immediately unlock instead of - waiting for a lock timeout. - - To make sure both sides of the lock know about the connection state it is - recommended to set the `health_check_interval` when creating the redis - connection.. - - Args: - channel: the redis channel to use as locking key. - connection: an optional redis connection if you already have one - or if you need to specify the redis connection - timeout: timeout when trying to acquire a lock - check_interval: check interval while waiting - fail_when_locked: after the initial lock failed, return an error - or lock the file. This does not wait for the timeout. - thread_sleep_time: sleep time between fetching messages from redis to - prevent a busy/wait loop. In the case of lock conflicts this - increases the time it takes to resolve the conflict. This should - be smaller than the `check_interval` to be useful. - unavailable_timeout: If the conflicting lock is properly connected - this should never exceed twice your redis latency. Note that this - will increase the wait time possibly beyond your `timeout` and is - always executed if a conflict arises. - redis_kwargs: The redis connection arguments if no connection is - given. The `DEFAULT_REDIS_KWARGS` are used as default, if you want - to override these you need to explicitly specify a value (e.g. - `health_check_interval=0`) - - """ - - redis_kwargs: dict[str, typing.Any] - thread: PubSubWorkerThread | None - channel: str - timeout: float - connection: redis.client.Redis[str] | None - pubsub: redis.client.PubSub | None = None - close_connection: bool - - DEFAULT_REDIS_KWARGS: typing.ClassVar[dict[str, typing.Any]] = dict( - health_check_interval=10, - decode_responses=True, - ) - - def __init__( - self, - channel: str, - connection: redis.client.Redis[str] | None = None, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = False, - thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, - unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, - redis_kwargs: dict[str, typing.Any] | None = None, - ) -> None: - # We don't want to close connections given as an argument - self.close_connection = not connection - - self.thread = None - self.channel = channel - self.connection = connection - self.thread_sleep_time = thread_sleep_time - self.unavailable_timeout = unavailable_timeout - self.redis_kwargs = redis_kwargs or dict() - - for key, value in self.DEFAULT_REDIS_KWARGS.items(): - self.redis_kwargs.setdefault(key, value) - - super().__init__( - timeout=timeout, - check_interval=check_interval, - fail_when_locked=fail_when_locked, - ) - - def get_connection(self) -> redis.client.Redis[str]: - if not self.connection: - self.connection = redis.client.Redis(**self.redis_kwargs) - - return self.connection - - def channel_handler(self, message: dict[str, str]) -> None: - if message.get('type') != 'message': # pragma: no cover - return - - raw_data = message.get('data') - if not raw_data: - return - - try: - data = json.loads(raw_data) - except TypeError: # pragma: no cover - logger.debug('TypeError while parsing: %r', message) - return - - assert self.connection is not None - self.connection.publish(data['response_channel'], str(time.time())) - - @property - def client_name(self) -> str: - return f'{self.channel}-lock' - - def _timeout_generator( - self, timeout: float | None, check_interval: float | None - ) -> typing.Iterator[int]: - if timeout is None: - timeout = 0.0 - if check_interval is None: - check_interval = self.thread_sleep_time - deadline = time.monotonic() + timeout - first = True - while first or time.monotonic() < deadline: - first = False - effective_interval = ( - check_interval - if check_interval > 0 - else self.thread_sleep_time - ) - sleep_time = effective_interval * (0.5 + random.random()) - time.sleep(sleep_time) - yield 0 - - def acquire( # type: ignore[override] - self, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = None, - ) -> RedisLock: - timeout = utils.coalesce(timeout, self.timeout, 0.0) - check_interval = utils.coalesce( - check_interval, - self.check_interval, - 0.0, - ) - fail_when_locked = utils.coalesce( - fail_when_locked, - self.fail_when_locked, - ) - - assert not self.pubsub, 'This lock is already active' - connection = self.get_connection() - - timeout_generator = self._timeout_generator(timeout, check_interval) - for _ in timeout_generator: # pragma: no branch - subscribers = connection.pubsub_numsub(self.channel)[0][1] - - if subscribers: - logger.debug( - 'Found %d lock subscribers for %s', - subscribers, - self.channel, - ) - - if self.check_or_kill_lock( - connection, - self.unavailable_timeout, - ): # pragma: no branch - continue - else: # pragma: no cover - subscribers = 0 - - # Note: this should not be changed to an elif because the if - # above can still end up here - if not subscribers: - connection.client_setname(self.client_name) - self.pubsub = connection.pubsub() - self.pubsub.subscribe(**{self.channel: self.channel_handler}) - self.thread = PubSubWorkerThread( - self.pubsub, - sleep_time=self.thread_sleep_time, - ) - self.thread.start() - time.sleep(0.01) - subscribers = connection.pubsub_numsub(self.channel)[0][1] - if subscribers == 1: # pragma: no branch - return self - else: # pragma: no cover - # Race condition, let's try again - self.release() - - if fail_when_locked: # pragma: no cover - raise exceptions.AlreadyLocked() - - raise exceptions.AlreadyLocked() - - def check_or_kill_lock( - self, - connection: redis.client.Redis[str], - timeout: float, - ) -> bool | None: - # Random channel name to get messages back from the lock - response_channel = f'{self.channel}-{random.random()}' - - pubsub = connection.pubsub() - pubsub.subscribe(response_channel) - connection.publish( - self.channel, - json.dumps( - dict( - response_channel=response_channel, - message='ping', - ), - ), - ) - - check_interval = min(self.thread_sleep_time, timeout / 10) - for _ in self._timeout_generator( - timeout, - check_interval, - ): # pragma: no branch - if pubsub.get_message(timeout=check_interval): - pubsub.close() - return True - - for client_ in connection.client_list('pubsub'): # pragma: no cover - if client_.get('name') == self.client_name: - logger.warning('Killing unavailable redis client: %r', client_) - connection.client_kill_filter( # pyright: ignore - client_.get('id'), - ) - return None - - def release(self) -> None: - if self.thread: # pragma: no branch - self.thread.stop() - self.thread.join() - self.thread = None - time.sleep(0.01) - - if self.pubsub: # pragma: no branch - self.pubsub.unsubscribe(self.channel) - self.pubsub.close() - self.pubsub = None - - def __del__(self) -> None: - self.release() diff --git a/contrib/python/portalocker/py3/portalocker/types.py b/contrib/python/portalocker/py3/portalocker/types.py deleted file mode 100644 index c08d426e0c2..00000000000 --- a/contrib/python/portalocker/py3/portalocker/types.py +++ /dev/null @@ -1,75 +0,0 @@ -# noqa: A005 -from __future__ import annotations - -import io -import pathlib -import typing -from typing import Union - -# spellchecker: off -# fmt: off -Mode = typing.Literal[ - # Text modes - # Read text - 'r', 'rt', 'tr', - # Write text - 'w', 'wt', 'tw', - # Append text - 'a', 'at', 'ta', - # Exclusive creation text - 'x', 'xt', 'tx', - # Read and write text - 'r+', '+r', 'rt+', 'r+t', '+rt', 'tr+', 't+r', '+tr', - # Write and read text - 'w+', '+w', 'wt+', 'w+t', '+wt', 'tw+', 't+w', '+tw', - # Append and read text - 'a+', '+a', 'at+', 'a+t', '+at', 'ta+', 't+a', '+ta', - # Exclusive creation and read text - 'x+', '+x', 'xt+', 'x+t', '+xt', 'tx+', 't+x', '+tx', - # Universal newline support - 'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr', - - # Binary modes - # Read binary - 'rb', 'br', - # Write binary - 'wb', 'bw', - # Append binary - 'ab', 'ba', - # Exclusive creation binary - 'xb', 'bx', - # Read and write binary - 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br', - # Write and read binary - 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw', - # Append and read binary - 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba', - # Exclusive creation and read binary - 'xb+', 'x+b', '+xb', 'bx+', 'b+x', '+bx', - # Universal newline support in binary mode - 'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr', -] -# spellchecker: on -Filename = Union[str, pathlib.Path] -IO = Union[ # type: ignore[name-defined] - typing.IO[str], - typing.IO[bytes], -] - - -class FileOpenKwargs(typing.TypedDict): - buffering: int | None - encoding: str | None - errors: str | None - newline: str | None - closefd: bool | None - opener: typing.Callable[[str, int], int] | None - - -# Protocol for objects with a fileno() method. -# Used for type-hinting fcntl.flock. -class HasFileno(typing.Protocol): - def fileno(self) -> int: ... - -# Type alias for file arguments used in lock/unlock functions -FileArgument = Union[typing.IO[typing.Any], io.TextIOWrapper, int, HasFileno] diff --git a/contrib/python/portalocker/py3/portalocker/utils.py b/contrib/python/portalocker/py3/portalocker/utils.py deleted file mode 100644 index b6dccb1404b..00000000000 --- a/contrib/python/portalocker/py3/portalocker/utils.py +++ /dev/null @@ -1,586 +0,0 @@ -from __future__ import annotations - -import abc -import atexit -import contextlib -import logging -import os -import pathlib -import random -import tempfile -import time -import typing -import warnings - -from . import constants, exceptions, portalocker, types -from .types import Filename, Mode - -logger = logging.getLogger(__name__) - -DEFAULT_TIMEOUT = 5 -DEFAULT_CHECK_INTERVAL = 0.25 -DEFAULT_FAIL_WHEN_LOCKED = False -LOCK_METHOD = constants.LockFlags.EXCLUSIVE | constants.LockFlags.NON_BLOCKING - -__all__ = [ - 'Lock', - 'open_atomic', -] - - -def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: - """Simple coalescing function that returns the first value that is not - equal to the `test_value`. Or `None` if no value is valid. Usually this - means that the last given value is the default value. - - Note that the `test_value` is compared using an identity check - (i.e. `value is not test_value`) so changing the `test_value` won't work - for all values. - - >>> coalesce(None, 1) - 1 - >>> coalesce() - - >>> coalesce(0, False, True) - 0 - >>> coalesce(0, False, True, test_value=0) - False - - # This won't work because of the `is not test_value` type testing: - >>> coalesce([], dict(spam='eggs'), test_value=[]) - [] - """ - return next((arg for arg in args if arg is not test_value), None) - - -def open_atomic( - filename: Filename, - binary: bool = True, -) -> typing.Iterator[types.IO]: - """Open a file for atomic writing. Instead of locking this method allows - you to write the entire file and move it to the actual location. Note that - this makes the assumption that a rename is atomic on your platform which - is generally the case but not a guarantee. - - http://docs.python.org/library/os.html#os.rename - - >>> filename = 'test_file.txt' - >>> if os.path.exists(filename): - ... os.remove(filename) - - >>> with open_atomic(filename) as fh: - ... written = fh.write(b'test') - >>> assert os.path.exists(filename) - >>> os.remove(filename) - - >>> import pathlib - >>> path_filename = pathlib.Path('test_file.txt') - - >>> with open_atomic(path_filename) as fh: - ... written = fh.write(b'test') - >>> assert path_filename.exists() - >>> path_filename.unlink() - """ - # `pathlib.Path` cast in case `path` is a `str` - path: pathlib.Path - if isinstance(filename, pathlib.Path): - path = filename - else: - path = pathlib.Path(filename) - - assert not path.exists(), f'{path!r} exists' - - # Create the parent directory if it doesn't exist - path.parent.mkdir(parents=True, exist_ok=True) - - with tempfile.NamedTemporaryFile( - mode=(binary and 'wb') or 'w', - dir=str(path.parent), - delete=False, - ) as temp_fh: - yield temp_fh - temp_fh.flush() - os.fsync(temp_fh.fileno()) - - try: - os.rename(temp_fh.name, path) - finally: - with contextlib.suppress(Exception): - os.remove(temp_fh.name) - - -class LockBase(abc.ABC): # pragma: no cover - #: timeout when trying to acquire a lock - timeout: float - #: check interval while waiting for `timeout` - check_interval: float - #: skip the timeout and immediately fail if the initial lock fails - fail_when_locked: bool - - def __init__( - self, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = None, - ) -> None: - self.timeout = coalesce(timeout, DEFAULT_TIMEOUT) - self.check_interval = coalesce(check_interval, DEFAULT_CHECK_INTERVAL) - self.fail_when_locked = coalesce( - fail_when_locked, - DEFAULT_FAIL_WHEN_LOCKED, - ) - - @abc.abstractmethod - def acquire( - self, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = None, - ) -> typing.IO[typing.AnyStr]: ... - - def _timeout_generator( - self, - timeout: float | None, - check_interval: float | None, - ) -> typing.Iterator[int]: - f_timeout = coalesce(timeout, self.timeout, 0.0) - f_check_interval = coalesce(check_interval, self.check_interval, 0.0) - - yield 0 - i = 0 - - start_time = time.perf_counter() - while start_time + f_timeout > time.perf_counter(): - i += 1 - yield i - - # Take low lock checks into account to stay within the interval - since_start_time = time.perf_counter() - start_time - time.sleep(max(0.001, (i * f_check_interval) - since_start_time)) - - @abc.abstractmethod - def release(self) -> None: ... - - def __enter__(self) -> typing.IO[typing.AnyStr]: - return self.acquire() - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: typing.Any, # Should be typing.TracebackType - ) -> bool | None: - self.release() - return None - - def __delete__(self, instance: LockBase) -> None: - instance.release() - - -class Lock(LockBase): - """Lock manager with built-in timeout - - Args: - filename: filename - mode: the open mode, 'a' or 'ab' should be used for writing. When mode - contains `w` the file will be truncated to 0 bytes. - timeout: timeout when trying to acquire a lock - check_interval: check interval while waiting - fail_when_locked: after the initial lock failed, return an error - or lock the file. This does not wait for the timeout. - **file_open_kwargs: The kwargs for the `open(...)` call - - fail_when_locked is useful when multiple threads/processes can race - when creating a file. If set to true than the system will wait till - the lock was acquired and then return an AlreadyLocked exception. - - Note that the file is opened first and locked later. So using 'w' as - mode will result in truncate _BEFORE_ the lock is checked. - """ - - fh: types.IO | None - filename: str - mode: str - truncate: bool - timeout: float - check_interval: float - fail_when_locked: bool - flags: constants.LockFlags - file_open_kwargs: dict[str, typing.Any] - - def __init__( - self, - filename: Filename, - mode: Mode = 'a', - timeout: float | None = None, - check_interval: float = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, - flags: constants.LockFlags = LOCK_METHOD, - **file_open_kwargs: typing.Any, - ) -> None: - if 'w' in mode: - truncate = True - mode = typing.cast(Mode, mode.replace('w', 'a')) - else: - truncate = False - - if timeout is None: - timeout = DEFAULT_TIMEOUT - elif not (flags & constants.LockFlags.NON_BLOCKING): - warnings.warn( - 'timeout has no effect in blocking mode', - stacklevel=1, - ) - - self.fh = None - self.filename = str(filename) - self.mode = mode - self.truncate = truncate - self.flags = flags - self.file_open_kwargs = file_open_kwargs - super().__init__(timeout, check_interval, fail_when_locked) - - def acquire( - self, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = None, - ) -> typing.IO[typing.AnyStr]: - """Acquire the locked filehandle""" - - fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) - - if ( - not (self.flags & constants.LockFlags.NON_BLOCKING) - and timeout is not None - ): - warnings.warn( - 'timeout has no effect in blocking mode', - stacklevel=1, - ) - - # If we already have a filehandle, return it - fh = self.fh - if fh: - # Due to type invariance we need to cast the type - return typing.cast(typing.IO[typing.AnyStr], fh) - - # Get a new filehandler - fh = self._get_fh() - - def try_close() -> None: # pragma: no cover - # Silently try to close the handle if possible, ignore all issues - if fh is not None: - with contextlib.suppress(Exception): - fh.close() - - exception = None - # Try till the timeout has passed - for _ in self._timeout_generator(timeout, check_interval): - exception = None - try: - # Try to lock - fh = self._get_lock(fh) - break - except exceptions.LockException as exc: - # Python will automatically remove the variable from memory - # unless you save it in a different location - exception = exc - - # We already tried to the get the lock - # If fail_when_locked is True, stop trying - if fail_when_locked: - try_close() - raise exceptions.AlreadyLocked(exception) from exc - except Exception as exc: - # Something went wrong with the locking mechanism. - # Wrap in a LockException and re-raise: - try_close() - raise exceptions.LockException(exc) from exc - - # Wait a bit - - if exception: - try_close() - # We got a timeout... reraising - raise exception - - # Prepare the filehandle (truncate if needed) - fh = self._prepare_fh(fh) - - self.fh = fh - return typing.cast(typing.IO[typing.AnyStr], fh) - - def __enter__(self) -> typing.IO[typing.AnyStr]: - return self.acquire() - - def release(self) -> None: - """Releases the currently locked file handle""" - if self.fh: - portalocker.unlock(self.fh) - self.fh.close() - self.fh = None - - def _get_fh(self) -> types.IO: - """Get a new filehandle""" - return typing.cast( - types.IO, - open( # noqa: SIM115 - self.filename, - self.mode, - **self.file_open_kwargs, - ), - ) - - def _get_lock(self, fh: types.IO) -> types.IO: - """ - Try to lock the given filehandle - - returns LockException if it fails""" - portalocker.lock(fh, self.flags) - return fh - - def _prepare_fh(self, fh: types.IO) -> types.IO: - """ - Prepare the filehandle for usage - - If truncate is a number, the file will be truncated to that amount of - bytes - """ - if self.truncate: - fh.seek(0) - fh.truncate(0) - - return fh - - -class RLock(Lock): - """ - A reentrant lock, functions in a similar way to threading.RLock in that it - can be acquired multiple times. When the corresponding number of release() - calls are made the lock will finally release the underlying file lock. - """ - - def __init__( - self, - filename: Filename, - mode: Mode = 'a', - timeout: float = DEFAULT_TIMEOUT, - check_interval: float = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool = False, - flags: constants.LockFlags = LOCK_METHOD, - ) -> None: - super().__init__( - filename, - mode, - timeout, - check_interval, - fail_when_locked, - flags, - ) - self._acquire_count = 0 - - def acquire( - self, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = None, - ) -> typing.IO[typing.AnyStr]: - fh: typing.IO[typing.AnyStr] - if self._acquire_count >= 1: - fh = typing.cast(typing.IO[typing.AnyStr], self.fh) - else: - fh = super().acquire(timeout, check_interval, fail_when_locked) - self._acquire_count += 1 - assert fh is not None - return fh - - def release(self) -> None: - if self._acquire_count == 0: - raise exceptions.LockException( - 'Cannot release more times than acquired', - ) - - if self._acquire_count == 1: - super().release() - self._acquire_count -= 1 - - -class TemporaryFileLock(Lock): - def __init__( - self, - filename: str = '.lock', - timeout: float = DEFAULT_TIMEOUT, - check_interval: float = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool = True, - flags: constants.LockFlags = LOCK_METHOD, - ) -> None: - super().__init__( - filename=filename, - mode='w', - timeout=timeout, - check_interval=check_interval, - fail_when_locked=fail_when_locked, - flags=flags, - ) - atexit.register(self.release) - - def release(self) -> None: - Lock.release(self) - if os.path.isfile(self.filename): # pragma: no branch - os.unlink(self.filename) - - -class BoundedSemaphore(LockBase): - """ - Bounded semaphore to prevent too many parallel processes from running - - This method is deprecated because multiple processes that are completely - unrelated could end up using the same semaphore. To prevent this, - use `NamedBoundedSemaphore` instead. The - `NamedBoundedSemaphore` is a drop-in replacement for this class. - - >>> semaphore = BoundedSemaphore(2, directory='') - >>> str(semaphore.get_filenames()[0]) - 'bounded_semaphore.00.lock' - >>> str(sorted(semaphore.get_random_filenames())[1]) - 'bounded_semaphore.01.lock' - """ - - lock: Lock | None - - def __init__( - self, - maximum: int, - name: str = 'bounded_semaphore', - filename_pattern: str = '{name}.{number:02d}.lock', - directory: str = tempfile.gettempdir(), - timeout: float | None = DEFAULT_TIMEOUT, - check_interval: float | None = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool | None = True, - ) -> None: - self.maximum = maximum - self.name = name - self.filename_pattern = filename_pattern - self.directory = directory - self.lock: Lock | None = None - super().__init__( - timeout=timeout, - check_interval=check_interval, - fail_when_locked=fail_when_locked, - ) - - if not name or name == 'bounded_semaphore': - warnings.warn( - '`BoundedSemaphore` without an explicit `name` ' - 'argument is deprecated, use NamedBoundedSemaphore', - DeprecationWarning, - stacklevel=1, - ) - - def get_filenames(self) -> typing.Sequence[pathlib.Path]: - return [self.get_filename(n) for n in range(self.maximum)] - - def get_random_filenames(self) -> typing.Sequence[pathlib.Path]: - filenames = list(self.get_filenames()) - random.shuffle(filenames) - return filenames - - def get_filename(self, number: int) -> pathlib.Path: - return pathlib.Path(self.directory) / self.filename_pattern.format( - name=self.name, - number=number, - ) - - def acquire( # type: ignore[override] - self, - timeout: float | None = None, - check_interval: float | None = None, - fail_when_locked: bool | None = None, - ) -> Lock | None: - assert not self.lock, 'Already locked' - - filenames = self.get_filenames() - - for n in self._timeout_generator(timeout, check_interval): # pragma: - logger.debug('trying lock (attempt %d) %r', n, filenames) - # no branch - if self.try_lock(filenames): # pragma: no branch - return self.lock # pragma: no cover - - if fail_when_locked := coalesce( - fail_when_locked, - self.fail_when_locked, - ): - raise exceptions.AlreadyLocked() - - return None - - def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: - filename: Filename - for filename in filenames: - logger.debug('trying lock for %r', filename) - self.lock = Lock(filename, fail_when_locked=True) - try: - self.lock.acquire() - except exceptions.AlreadyLocked: - self.lock = None - else: - logger.debug('locked %r', filename) - return True - - return False - - def release(self) -> None: # pragma: no cover - if self.lock is not None: - self.lock.release() - self.lock = None - - -class NamedBoundedSemaphore(BoundedSemaphore): - """ - Bounded semaphore to prevent too many parallel processes from running - - It's also possible to specify a timeout when acquiring the lock to wait - for a resource to become available. This is very similar to - `threading.BoundedSemaphore` but works across multiple processes and across - multiple operating systems. - - Because this works across multiple processes it's important to give the - semaphore a name. This name is used to create the lock files. If you - don't specify a name, a random name will be generated. This means that - you can't use the same semaphore in multiple processes unless you pass the - semaphore object to the other processes. - - >>> semaphore = NamedBoundedSemaphore(2, name='test') - >>> str(semaphore.get_filenames()[0]) - '...test.00.lock' - - >>> semaphore = NamedBoundedSemaphore(2) - >>> 'bounded_semaphore' in str(semaphore.get_filenames()[0]) - True - - """ - - def __init__( - self, - maximum: int, - name: str | None = None, - filename_pattern: str = '{name}.{number:02d}.lock', - directory: str = tempfile.gettempdir(), - timeout: float | None = DEFAULT_TIMEOUT, - check_interval: float | None = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool | None = True, - ) -> None: - if name is None: - name = f'bounded_semaphore.{random.randint(0, 1000000):d}' - super().__init__( - maximum, - name, - filename_pattern, - directory, - timeout, - check_interval, - fail_when_locked, - ) diff --git a/contrib/python/portalocker/py3/tests/__init__.py b/contrib/python/portalocker/py3/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/portalocker/py3/tests/__init__.py +++ /dev/null diff --git a/contrib/python/portalocker/py3/tests/conftest.py b/contrib/python/portalocker/py3/tests/conftest.py deleted file mode 100644 index cf59e2b32f4..00000000000 --- a/contrib/python/portalocker/py3/tests/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging -import pytest -import random -import multiprocessing - -logger = logging.getLogger(__name__) - - -def tmpfile(tmp_path): - filename = tmp_path / str(random.random()) - yield str(filename) - try: - filename.unlink(missing_ok=True) - except PermissionError: - pass - - -def pytest_sessionstart(session): - # Force spawning the process so we don't accidently inherit locks. - # I'm not a 100% certain this will work correctly unfortunately... there - # is some potential for breaking tests - multiprocessing.set_start_method('spawn') diff --git a/contrib/python/portalocker/py3/tests/temporary_file_lock.py b/contrib/python/portalocker/py3/tests/temporary_file_lock.py deleted file mode 100644 index b250bad6510..00000000000 --- a/contrib/python/portalocker/py3/tests/temporary_file_lock.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import portalocker - - -def test_temporary_file_lock(tmpfile): - with portalocker.TemporaryFileLock(tmpfile): - pass - - assert not os.path.isfile(tmpfile) - - lock = portalocker.TemporaryFileLock(tmpfile) - lock.acquire() - del lock - diff --git a/contrib/python/portalocker/py3/tests/test_combined.py b/contrib/python/portalocker/py3/tests/test_combined.py deleted file mode 100644 index 594de74b9ba..00000000000 --- a/contrib/python/portalocker/py3/tests/test_combined.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys - - -def test_combined(tmpdir): - from distutils import dist - import setup - - output_file = tmpdir.join('combined.py') - combine = setup.Combine(dist.Distribution()) - combine.output_file = str(output_file) - combine.run() - sys.path.append(output_file.dirname) - import combined - assert combined - diff --git a/contrib/python/portalocker/py3/tests/test_redis.py b/contrib/python/portalocker/py3/tests/test_redis.py deleted file mode 100644 index 694c9af17ca..00000000000 --- a/contrib/python/portalocker/py3/tests/test_redis.py +++ /dev/null @@ -1,90 +0,0 @@ -import _thread -import logging -import random -import time - -import pytest -from redis import client -from redis import exceptions - -import portalocker -from portalocker import redis -from portalocker import utils - -logger = logging.getLogger(__name__) - -try: - client.Redis().ping() -except (exceptions.ConnectionError, ConnectionRefusedError): - pytest.skip('Unable to connect to redis', allow_module_level=True) - - [email protected](autouse=True) -def set_redis_timeouts(monkeypatch): - monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) - monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) - monkeypatch.setattr(redis, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) - monkeypatch.setattr(redis, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) - monkeypatch.setattr(_thread, 'interrupt_main', lambda: None) - - -def test_redis_lock(): - channel = str(random.random()) - - lock_a = redis.RedisLock(channel) - lock_a.acquire(fail_when_locked=True) - time.sleep(0.01) - - lock_b = redis.RedisLock(channel) - try: - with pytest.raises(portalocker.AlreadyLocked): - lock_b.acquire(fail_when_locked=True) - finally: - lock_a.release() - lock_a.connection.close() - - [email protected]('timeout', [None, 0, 0.001]) [email protected]('check_interval', [None, 0, 0.0005]) -def test_redis_lock_timeout(timeout, check_interval): - connection = client.Redis() - channel = str(random.random()) - lock_a = redis.RedisLock(channel) - lock_a.acquire(timeout=timeout, check_interval=check_interval) - - lock_b = redis.RedisLock(channel, connection=connection) - with pytest.raises(portalocker.AlreadyLocked): - try: - lock_b.acquire(timeout=timeout, check_interval=check_interval) - finally: - lock_a.release() - lock_a.connection.close() - - -def test_redis_lock_context(): - channel = str(random.random()) - - lock_a = redis.RedisLock(channel, fail_when_locked=True) - with lock_a: - time.sleep(0.01) - lock_b = redis.RedisLock(channel, fail_when_locked=True) - with pytest.raises(portalocker.AlreadyLocked): - with lock_b: - pass - - -def test_redis_relock(): - channel = str(random.random()) - - lock_a = redis.RedisLock(channel, fail_when_locked=True) - with lock_a: - time.sleep(0.01) - with pytest.raises(AssertionError): - lock_a.acquire() - time.sleep(0.01) - - lock_a.release() - - -if __name__ == '__main__': - test_redis_lock() diff --git a/contrib/python/portalocker/py3/tests/test_semaphore.py b/contrib/python/portalocker/py3/tests/test_semaphore.py deleted file mode 100644 index b0c57aa23c7..00000000000 --- a/contrib/python/portalocker/py3/tests/test_semaphore.py +++ /dev/null @@ -1,22 +0,0 @@ -import random -import pytest -import portalocker -from portalocker import utils - - [email protected]('timeout', [None, 0, 0.001]) [email protected]('check_interval', [None, 0, 0.0005]) -def test_bounded_semaphore(timeout, check_interval, monkeypatch): - n = 2 - name = random.random() - monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) - monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) - - semaphore_a = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) - semaphore_b = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) - semaphore_c = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) - - semaphore_a.acquire(timeout=timeout) - semaphore_b.acquire() - with pytest.raises(portalocker.AlreadyLocked): - semaphore_c.acquire(check_interval=check_interval, timeout=timeout) diff --git a/contrib/python/portalocker/py3/tests/tests.py b/contrib/python/portalocker/py3/tests/tests.py deleted file mode 100644 index 7a00405966e..00000000000 --- a/contrib/python/portalocker/py3/tests/tests.py +++ /dev/null @@ -1,357 +0,0 @@ -from __future__ import print_function -from __future__ import with_statement - -import os -import dataclasses -import multiprocessing -import time -import typing - -import pytest -import portalocker -from portalocker import utils -from portalocker import LockFlags - - -def test_exceptions(tmpfile): - # Open the file 2 times - a = open(tmpfile, 'a') - b = open(tmpfile, 'a') - - # Lock exclusive non-blocking - lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB - - # First lock file a - portalocker.lock(a, lock_flags) - - # Now see if we can lock file b - with pytest.raises(portalocker.LockException): - portalocker.lock(b, lock_flags) - - # Cleanup - a.close() - b.close() - - -def test_utils_base(): - class Test(utils.LockBase): - pass - - -def test_with_timeout(tmpfile): - # Open the file 2 times - with pytest.raises(portalocker.AlreadyLocked): - with portalocker.Lock(tmpfile, timeout=0.1) as fh: - print('writing some stuff to my cache...', file=fh) - with portalocker.Lock( - tmpfile, timeout=0.1, mode='wb', - fail_when_locked=True - ): - pass - print('writing more stuff to my cache...', file=fh) - - -def test_without_timeout(tmpfile): - # Open the file 2 times - with pytest.raises(portalocker.LockException): - with portalocker.Lock(tmpfile, timeout=None) as fh: - print('writing some stuff to my cache...', file=fh) - with portalocker.Lock(tmpfile, timeout=None, mode='w'): - pass - print('writing more stuff to my cache...', file=fh) - - -def test_without_fail(tmpfile): - # Open the file 2 times - with pytest.raises(portalocker.LockException): - with portalocker.Lock(tmpfile, timeout=0.1) as fh: - print('writing some stuff to my cache...', file=fh) - lock = portalocker.Lock(tmpfile, timeout=0.1) - lock.acquire(check_interval=0.05, fail_when_locked=False) - - -def test_simple(tmpfile): - with open(tmpfile, 'w') as fh: - fh.write('spam and eggs') - - fh = open(tmpfile, 'r+') - portalocker.lock(fh, portalocker.LOCK_EX) - - fh.seek(13) - fh.write('foo') - - # Make sure we didn't overwrite the original text - fh.seek(0) - assert fh.read(13) == 'spam and eggs' - - portalocker.unlock(fh) - fh.close() - - -def test_truncate(tmpfile): - with open(tmpfile, 'w') as fh: - fh.write('spam and eggs') - - with portalocker.Lock(tmpfile, mode='a+') as fh: - # Make sure we didn't overwrite the original text - fh.seek(0) - assert fh.read(13) == 'spam and eggs' - - with portalocker.Lock(tmpfile, mode='w+') as fh: - # Make sure we truncated the file - assert fh.read() == '' - - -def test_class(tmpfile): - lock = portalocker.Lock(tmpfile) - lock2 = portalocker.Lock(tmpfile, fail_when_locked=False, timeout=0.01) - - with lock: - lock.acquire() - - with pytest.raises(portalocker.LockException): - with lock2: - pass - - with lock2: - pass - - -def test_acquire_release(tmpfile): - lock = portalocker.Lock(tmpfile) - lock2 = portalocker.Lock(tmpfile, fail_when_locked=False) - - lock.acquire() # acquire lock when nobody is using it - with pytest.raises(portalocker.LockException): - # another party should not be able to acquire the lock - lock2.acquire(timeout=0.01) - - # re-acquire a held lock is a no-op - lock.acquire() - - lock.release() # release the lock - lock.release() # second release does nothing - - -def test_rlock_acquire_release_count(tmpfile): - lock = portalocker.RLock(tmpfile) - # Twice acquire - h = lock.acquire() - assert not h.closed - lock.acquire() - assert not h.closed - - # Two release - lock.release() - assert not h.closed - lock.release() - assert h.closed - - -def test_rlock_acquire_release(tmpfile): - lock = portalocker.RLock(tmpfile) - lock2 = portalocker.RLock(tmpfile, fail_when_locked=False) - - lock.acquire() # acquire lock when nobody is using it - with pytest.raises(portalocker.LockException): - # another party should not be able to acquire the lock - lock2.acquire(timeout=0.01) - - # Now acquire again - lock.acquire() - - lock.release() # release the lock - lock.release() # second release does nothing - - -def test_release_unacquired(tmpfile): - with pytest.raises(portalocker.LockException): - portalocker.RLock(tmpfile).release() - - -def test_exlusive(tmpfile): - with open(tmpfile, 'w') as fh: - fh.write('spam and eggs') - - fh = open(tmpfile, 'r') - portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) - - # Make sure we can't read the locked file - with pytest.raises(portalocker.LockException): - with open(tmpfile, 'r') as fh2: - portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) - fh2.read() - - # Make sure we can't write the locked file - with pytest.raises(portalocker.LockException): - with open(tmpfile, 'w+') as fh2: - portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) - fh2.write('surprise and fear') - - # Make sure we can explicitly unlock the file - portalocker.unlock(fh) - fh.close() - - -def test_shared(tmpfile): - with open(tmpfile, 'w') as fh: - fh.write('spam and eggs') - - f = open(tmpfile, 'r') - portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB) - - # Make sure we can read the locked file - with open(tmpfile, 'r') as fh2: - portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB) - assert fh2.read() == 'spam and eggs' - - # Make sure we can't write the locked file - with pytest.raises(portalocker.LockException): - with open(tmpfile, 'w+') as fh2: - portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) - fh2.write('surprise and fear') - - # Make sure we can explicitly unlock the file - portalocker.unlock(f) - f.close() - - -def test_blocking_timeout(tmpfile): - flags = LockFlags.SHARED - - with pytest.warns(UserWarning): - with portalocker.Lock(tmpfile, timeout=5, flags=flags): - pass - - lock = portalocker.Lock(tmpfile, flags=flags) - with pytest.warns(UserWarning): - lock.acquire(timeout=5) - - [email protected](os.name == 'nt', - reason='Windows uses an entirely different lockmechanism') -def test_nonblocking(tmpfile): - with open(tmpfile, 'w') as fh: - with pytest.raises(RuntimeError): - portalocker.lock(fh, LockFlags.NON_BLOCKING) - - -def shared_lock(filename, **kwargs): - with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=False, - flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, - ): - time.sleep(0.2) - return True - - -def shared_lock_fail(filename, **kwargs): - with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=True, - flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, - ): - time.sleep(0.2) - return True - - -def exclusive_lock(filename, **kwargs): - with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=False, - flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, - ): - time.sleep(0.2) - return True - - [email protected](order=True) -class LockResult: - exception_class: typing.Union[typing.Type[Exception], None] = None - exception_message: typing.Union[str, None] = None - exception_repr: typing.Union[str, None] = None - - -def lock( - filename: str, - fail_when_locked: bool, - flags: LockFlags -) -> LockResult: - # Returns a case of True, False or FileNotFound - # https://thedailywtf.com/articles/what_is_truth_0x3f_ - # But seriously, the exception properties cannot be safely pickled so we - # only return string representations of the exception properties - try: - with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=fail_when_locked, - flags=flags, - ): - time.sleep(0.2) - return LockResult() - - except Exception as exception: - # The exceptions cannot be pickled so we cannot return them through - # multiprocessing - return LockResult( - type(exception), - str(exception), - repr(exception), - ) - - [email protected]('fail_when_locked', [True, False]) -def test_shared_processes(tmpfile, fail_when_locked): - flags = LockFlags.SHARED | LockFlags.NON_BLOCKING - - with multiprocessing.Pool(processes=2) as pool: - args = tmpfile, fail_when_locked, flags - results = pool.starmap_async(lock, 2 * [args]) - - for result in results.get(timeout=3): - assert result == LockResult() - - [email protected]('fail_when_locked', [True, False]) -def test_exclusive_processes(tmpfile, fail_when_locked): - flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING - - with multiprocessing.Pool(processes=2) as pool: - # filename, fail_when_locked, flags - args = tmpfile, fail_when_locked, flags - a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=3) - - assert not a.exception_class or not b.exception_class - assert issubclass( - a.exception_class or b.exception_class, - portalocker.LockException - ) - - - os.name == 'nt', - reason='Locking on Windows requires a file object', -) -def test_lock_fileno(tmpfile): - # Open the file 2 times - a = open(tmpfile, 'a') - b = open(tmpfile, 'a') - - # Lock exclusive non-blocking - flags = LockFlags.SHARED | LockFlags.NON_BLOCKING - - # First lock file a - portalocker.lock(a, flags) - - # Now see if we can lock using fileno() - portalocker.lock(b.fileno(), flags) - - # Cleanup - a.close() - b.close() - diff --git a/contrib/python/portalocker/py3/tests/ya.make b/contrib/python/portalocker/py3/tests/ya.make deleted file mode 100644 index 342f72a5391..00000000000 --- a/contrib/python/portalocker/py3/tests/ya.make +++ /dev/null @@ -1,23 +0,0 @@ -PY3TEST() - -SUBSCRIBER(g:python-contrib) - -NO_LINT() - -PEERDIR( - contrib/python/portalocker - contrib/python/redis -) - -TEST_SRCS( - __init__.py - conftest.py - temporary_file_lock.py - # Tests intallation. - # test_combined.py - test_redis.py - test_semaphore.py - tests.py -) - -END() diff --git a/contrib/python/portalocker/py3/ya.make b/contrib/python/portalocker/py3/ya.make deleted file mode 100644 index 0929f09336f..00000000000 --- a/contrib/python/portalocker/py3/ya.make +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(3.2.0) - -LICENSE(BSD-3-Clause) - -NO_LINT() - -NO_CHECK_IMPORTS( - portalocker.redis -) - -PY_SRCS( - TOP_LEVEL - portalocker/__about__.py - portalocker/__init__.py - portalocker/__main__.py - portalocker/constants.py - portalocker/exceptions.py - portalocker/portalocker.py - portalocker/redis.py - portalocker/types.py - portalocker/utils.py -) - -RESOURCE_FILES( - PREFIX contrib/python/portalocker/py3/ - .dist-info/METADATA - .dist-info/top_level.txt - portalocker/py.typed -) - -END() - -RECURSE( - tests -) |