diff options
Diffstat (limited to 'contrib/python')
256 files changed, 46921 insertions, 0 deletions
diff --git a/contrib/python/black/.dist-info/METADATA b/contrib/python/black/.dist-info/METADATA new file mode 100644 index 00000000000..64273a7e12c --- /dev/null +++ b/contrib/python/black/.dist-info/METADATA @@ -0,0 +1,2203 @@ +Metadata-Version: 2.4 +Name: black +Version: 25.1.0 +Summary: The uncompromising code formatter. +Project-URL: Documentation, https://black.readthedocs.io/ +Project-URL: Changelog, https://github.com/psf/black/blob/main/CHANGES.md +Project-URL: Repository, https://github.com/psf/black +Project-URL: Issues, https://github.com/psf/black/issues +Author-email: Łukasz Langa <[email protected]> +License-Expression: MIT +License-File: AUTHORS.md +License-File: LICENSE +Keywords: automation,autopep8,formatter,gofmt,pyfmt,rustfmt,yapf +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Quality Assurance +Requires-Python: >=3.9 +Requires-Dist: click>=8.0.0 +Requires-Dist: mypy-extensions>=0.4.3 +Requires-Dist: packaging>=22.0 +Requires-Dist: pathspec>=0.9.0 +Requires-Dist: platformdirs>=2 +Requires-Dist: tomli>=1.1.0; python_version < '3.11' +Requires-Dist: typing-extensions>=4.0.1; python_version < '3.11' +Provides-Extra: colorama +Requires-Dist: colorama>=0.4.3; extra == 'colorama' +Provides-Extra: d +Requires-Dist: aiohttp>=3.10; extra == 'd' +Provides-Extra: jupyter +Requires-Dist: ipython>=7.8.0; extra == 'jupyter' +Requires-Dist: tokenize-rt>=3.2.0; extra == 'jupyter' +Provides-Extra: uvloop +Requires-Dist: uvloop>=0.15.2; extra == 'uvloop' +Description-Content-Type: text/markdown + +[](https://black.readthedocs.io/en/stable/) + +<h2 align="center">The Uncompromising Code Formatter</h2> + +<p align="center"> +<a href="https://github.com/psf/black/actions"><img alt="Actions Status" src="https://github.com/psf/black/workflows/Test/badge.svg"></a> +<a href="https://black.readthedocs.io/en/stable/?badge=stable"><img alt="Documentation Status" src="https://readthedocs.org/projects/black/badge/?version=stable"></a> +<a href="https://coveralls.io/github/psf/black?branch=main"><img alt="Coverage Status" src="https://coveralls.io/repos/github/psf/black/badge.svg?branch=main"></a> +<a href="https://github.com/psf/black/blob/main/LICENSE"><img alt="License: MIT" src="https://black.readthedocs.io/en/stable/_static/license.svg"></a> +<a href="https://pypi.org/project/black/"><img alt="PyPI" src="https://img.shields.io/pypi/v/black"></a> +<a href="https://pepy.tech/project/black"><img alt="Downloads" src="https://static.pepy.tech/badge/black"></a> +<a href="https://anaconda.org/conda-forge/black/"><img alt="conda-forge" src="https://img.shields.io/conda/dn/conda-forge/black.svg?label=conda-forge"></a> +<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a> +</p> + +> “Any color you like.” + +_Black_ is the uncompromising Python code formatter. By using it, you agree to cede +control over minutiae of hand-formatting. In return, _Black_ gives you speed, +determinism, and freedom from `pycodestyle` nagging about formatting. You will save time +and mental energy for more important matters. + +Blackened code looks the same regardless of the project you're reading. Formatting +becomes transparent after a while and you can focus on the content instead. + +_Black_ makes code review faster by producing the smallest diffs possible. + +Try it out now using the [Black Playground](https://black.vercel.app). Watch the +[PyCon 2019 talk](https://youtu.be/esZLCuWs_2Y) to learn more. + +--- + +**[Read the documentation on ReadTheDocs!](https://black.readthedocs.io/en/stable)** + +--- + +## Installation and usage + +### Installation + +_Black_ can be installed by running `pip install black`. It requires Python 3.9+ to run. +If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. + +If you can't wait for the latest _hotness_ and want to install from GitHub, use: + +`pip install git+https://github.com/psf/black` + +### Usage + +To get started right away with sensible defaults: + +```sh +black {source_file_or_directory} +``` + +You can run _Black_ as a package if running it as a script doesn't work: + +```sh +python -m black {source_file_or_directory} +``` + +Further information can be found in our docs: + +- [Usage and Configuration](https://black.readthedocs.io/en/stable/usage_and_configuration/index.html) + +_Black_ is already [successfully used](https://github.com/psf/black#used-by) by many +projects, small and big. _Black_ has a comprehensive test suite, with efficient parallel +tests, and our own auto formatting and parallel Continuous Integration runner. Now that +we have become stable, you should not expect large formatting changes in the future. +Stylistic changes will mostly be responses to bug reports and support for new Python +syntax. For more information please refer to +[The Black Code Style](https://black.readthedocs.io/en/stable/the_black_code_style/index.html). + +Also, as a safety measure which slows down processing, _Black_ will check that the +reformatted code still produces a valid AST that is effectively equivalent to the +original (see the +[Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#ast-before-and-after-formatting) +section for details). If you're feeling confident, use `--fast`. + +## The _Black_ code style + +_Black_ is a PEP 8 compliant opinionated formatter. _Black_ reformats entire files in +place. Style configuration options are deliberately limited and rarely added. It doesn't +take previous formatting into account (see +[Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#pragmatism) +for exceptions). + +Our documentation covers the current _Black_ code style, but planned changes to it are +also documented. They're both worth taking a look at: + +- [The _Black_ Code Style: Current style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) +- [The _Black_ Code Style: Future style](https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html) + +Changes to the _Black_ code style are bound by the Stability Policy: + +- [The _Black_ Code Style: Stability Policy](https://black.readthedocs.io/en/stable/the_black_code_style/index.html#stability-policy) + +Please refer to this document before submitting an issue. What seems like a bug might be +intended behaviour. + +### Pragmatism + +Early versions of _Black_ used to be absolutist in some respects. They took after its +initial author. This was fine at the time as it made the implementation simpler and +there were not many users anyway. Not many edge cases were reported. As a mature tool, +_Black_ does make some exceptions to rules it otherwise holds. + +- [The _Black_ code style: Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#pragmatism) + +Please refer to this document before submitting an issue just like with the document +above. What seems like a bug might be intended behaviour. + +## Configuration + +_Black_ is able to read project-specific default values for its command line options +from a `pyproject.toml` file. This is especially useful for specifying custom +`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your +project. + +You can find more details in our documentation: + +- [The basics: Configuration via a file](https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-via-a-file) + +And if you're looking for more general configuration documentation: + +- [Usage and Configuration](https://black.readthedocs.io/en/stable/usage_and_configuration/index.html) + +**Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is +"No". _Black_ is all about sensible defaults. Applying those defaults will have your +code in compliance with many other _Black_ formatted projects. + +## Used by + +The following notable open-source projects trust _Black_ with enforcing a consistent +code style: pytest, tox, Pyramid, Django, Django Channels, Hypothesis, attrs, +SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtualenv), +pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, +Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. + +The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, +Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. + +Are we missing anyone? Let us know. + +## Testimonials + +**Mike Bayer**, [author of `SQLAlchemy`](https://www.sqlalchemy.org/): + +> I can't think of any single tool in my entire programming career that has given me a +> bigger productivity increase by its introduction. I can now do refactorings in about +> 1% of the keystrokes that it would have taken me previously when we had no way for +> code to format itself. + +**Dusty Phillips**, +[writer](https://smile.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords=dusty+phillips): + +> _Black_ is opinionated so you don't have to be. + +**Hynek Schlawack**, [creator of `attrs`](https://www.attrs.org/), core developer of +Twisted and CPython: + +> An auto-formatter that doesn't suck is all I want for Xmas! + +**Carl Meyer**, [Django](https://www.djangoproject.com/) core developer: + +> At least the name is good. + +**Kenneth Reitz**, creator of [`requests`](https://requests.readthedocs.io/en/latest/) +and [`pipenv`](https://readthedocs.org/projects/pipenv/): + +> This vastly improves the formatting of our code. Thanks a ton! + +## Show your style + +Use the badge in your project's README.md: + +```md +[](https://github.com/psf/black) +``` + +Using the badge in README.rst: + +``` +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black +``` + +Looks like this: +[](https://github.com/psf/black) + +## License + +MIT + +## Contributing + +Welcome! Happy to see you willing to make the project better. You can get started by +reading this: + +- [Contributing: The basics](https://black.readthedocs.io/en/latest/contributing/the_basics.html) + +You can also take a look at the rest of the contributing docs or talk with the +developers: + +- [Contributing documentation](https://black.readthedocs.io/en/latest/contributing/index.html) +- [Chat on Discord](https://discord.gg/RtVdv86PrH) + +## Change log + +The log has become rather long. It moved to its own file. + +See [CHANGES](https://black.readthedocs.io/en/latest/change_log.html). + +## Authors + +The author list is quite long nowadays, so it lives in its own file. + +See [AUTHORS.md](./AUTHORS.md) + +## Code of Conduct + +Everyone participating in the _Black_ project, and in particular in the issue tracker, +pull requests, and social media activity, is expected to treat other people with respect +and more generally to follow the guidelines articulated in the +[Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/). + +At the same time, humor is encouraged. In fact, basic familiarity with Monty Python's +Flying Circus is expected. We are not savages. + +And if you _really_ need to slap somebody, do it with a fish while dancing. +# Change Log + +## 25.1.0 + +### Highlights + +This release introduces the new 2025 stable style (#4558), stabilizing +the following changes: + +- Normalize casing of Unicode escape characters in strings to lowercase (#2916) +- Fix inconsistencies in whether certain strings are detected as docstrings (#4095) +- Consistently add trailing commas to typed function parameters (#4164) +- Remove redundant parentheses in if guards for case blocks (#4214) +- Add parentheses to if clauses in case blocks when the line is too long (#4269) +- Whitespace before `# fmt: skip` comments is no longer normalized (#4146) +- Fix line length computation for certain expressions that involve the power operator (#4154) +- Check if there is a newline before the terminating quotes of a docstring (#4185) +- Fix type annotation spacing between `*` and more complex type variable tuple (#4440) + +The following changes were not in any previous release: + +- Remove parentheses around sole list items (#4312) +- Generic function definitions are now formatted more elegantly: parameters are + split over multiple lines first instead of type parameter definitions (#4553) + +### Stable style + +- Fix formatting cells in IPython notebooks with magic methods and starting or trailing + empty lines (#4484) +- Fix crash when formatting `with` statements containing tuple generators/unpacking + (#4538) + +### Preview style + +- Fix/remove string merging changing f-string quotes on f-strings with internal quotes + (#4498) +- Collapse multiple empty lines after an import into one (#4489) +- Prevent `string_processing` and `wrap_long_dict_values_in_parens` from removing + parentheses around long dictionary values (#4377) +- Move `wrap_long_dict_values_in_parens` from the unstable to preview style (#4561) + +### Packaging + +- Store license identifier inside the `License-Expression` metadata field, see + [PEP 639](https://peps.python.org/pep-0639/). (#4479) + +### Performance + +- Speed up the `is_fstring_start` function in Black's tokenizer (#4541) + +### Integrations + +- If using stdin with `--stdin-filename` set to a force excluded path, stdin won't be + formatted. (#4539) + +## 24.10.0 + +### Highlights + +- Black is now officially tested with Python 3.13 and provides Python 3.13 + mypyc-compiled wheels. (#4436) (#4449) +- Black will issue an error when used with Python 3.12.5, due to an upstream memory + safety issue in Python 3.12.5 that can cause Black's AST safety checks to fail. Please + use Python 3.12.6 or Python 3.12.4 instead. (#4447) +- Black no longer supports running with Python 3.8 (#4452) + +### Stable style + +- Fix crashes involving comments in parenthesised return types or `X | Y` style unions. + (#4453) +- Fix skipping Jupyter cells with unknown `%%` magic (#4462) + +### Preview style + +- Fix type annotation spacing between * and more complex type variable tuple (i.e. `def + fn(*args: *tuple[*Ts, T]) -> None: pass`) (#4440) + +### Caching + +- Fix bug where the cache was shared between runs with and without `--unstable` (#4466) + +### Packaging + +- Upgrade version of mypyc used to 1.12 beta (#4450) (#4449) +- `blackd` now requires a newer version of aiohttp. (#4451) + +### Output + +- Added Python target version information on parse error (#4378) +- Add information about Black version to internal error messages (#4457) + +## 24.8.0 + +### Stable style + +- Fix crash when `# fmt: off` is used before a closing parenthesis or bracket. (#4363) + +### Packaging + +- Packaging metadata updated: docs are explictly linked, the issue tracker is now also + linked. This improves the PyPI listing for Black. (#4345) + +### Parser + +- Fix regression where Black failed to parse a multiline f-string containing another + multiline string (#4339) +- Fix regression where Black failed to parse an escaped single quote inside an f-string + (#4401) +- Fix bug with Black incorrectly parsing empty lines with a backslash (#4343) +- Fix bugs with Black's tokenizer not handling `\{` inside f-strings very well (#4422) +- Fix incorrect line numbers in the tokenizer for certain tokens within f-strings + (#4423) + +### Performance + +- Improve performance when a large directory is listed in `.gitignore` (#4415) + +### _Blackd_ + +- Fix blackd (and all extras installs) for docker container (#4357) + +## 24.4.2 + +This is a bugfix release to fix two regressions in the new f-string parser introduced in +24.4.1. + +### Parser + +- Fix regression where certain complex f-strings failed to parse (#4332) + +### Performance + +- Fix bad performance on certain complex string literals (#4331) + +## 24.4.1 + +### Highlights + +- Add support for the new Python 3.12 f-string syntax introduced by PEP 701 (#3822) + +### Stable style + +- Fix crash involving indented dummy functions containing newlines (#4318) + +### Parser + +- Add support for type parameter defaults, a new syntactic feature added to Python 3.13 + by PEP 696 (#4327) + +### Integrations + +- Github Action now works even when `git archive` is skipped (#4313) + +## 24.4.0 + +### Stable style + +- Fix unwanted crashes caused by AST equivalency check (#4290) + +### Preview style + +- `if` guards in `case` blocks are now wrapped in parentheses when the line is too long. + (#4269) +- Stop moving multiline strings to a new line unless inside brackets (#4289) + +### Integrations + +- Add a new option `use_pyproject` to the GitHub Action `psf/black`. This will read the + Black version from `pyproject.toml`. (#4294) + +## 24.3.0 + +### Highlights + +This release is a milestone: it fixes Black's first CVE security vulnerability. If you +run Black on untrusted input, or if you habitually put thousands of leading tab +characters in your docstrings, you are strongly encouraged to upgrade immediately to fix +[CVE-2024-21503](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-21503). + +This release also fixes a bug in Black's AST safety check that allowed Black to make +incorrect changes to certain f-strings that are valid in Python 3.12 and higher. + +### Stable style + +- Don't move comments along with delimiters, which could cause crashes (#4248) +- Strengthen AST safety check to catch more unsafe changes to strings. Previous versions + of Black would incorrectly format the contents of certain unusual f-strings containing + nested strings with the same quote type. Now, Black will crash on such strings until + support for the new f-string syntax is implemented. (#4270) +- Fix a bug where line-ranges exceeding the last code line would not work as expected + (#4273) + +### Performance + +- Fix catastrophic performance on docstrings that contain large numbers of leading tab + characters. This fixes + [CVE-2024-21503](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-21503). + (#4278) + +### Documentation + +- Note what happens when `--check` is used with `--quiet` (#4236) + +## 24.2.0 + +### Stable style + +- Fixed a bug where comments where mistakenly removed along with redundant parentheses + (#4218) + +### Preview style + +- Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style + due to an outstanding crash and proposed formatting tweaks (#4198) +- Fixed a bug where base expressions caused inconsistent formatting of \*\* in tenary + expression (#4154) +- Checking for newline before adding one on docstring that is almost at the line limit + (#4185) +- Remove redundant parentheses in `case` statement `if` guards (#4214). + +### Configuration + +- Fix issue where _Black_ would ignore input files in the presence of symlinks (#4222) +- _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when + discovering project root and configuration. Since _Black_ continues to use version + control as an indicator of project root, this is expected to primarily change behavior + for users in a monorepo setup (desirably). If you wish to preserve previous behavior, + simply add an empty `[tool.black]` to the previously discovered `pyproject.toml` + (#4204) + +### Output + +- Black will swallow any `SyntaxWarning`s or `DeprecationWarning`s produced by the `ast` + module when performing equivalence checks (#4189) + +### Integrations + +- Add a JSONSchema and provide a validate-pyproject entry-point (#4181) + +## 24.1.1 + +Bugfix release to fix a bug that made Black unusable on certain file systems with strict +limits on path length. + +### Preview style + +- Consistently add trailing comma on typed parameters (#4164) + +### Configuration + +- Shorten the length of the name of the cache file to fix crashes on file systems that + do not support long paths (#4176) + +## 24.1.0 + +### Highlights + +This release introduces the new 2024 stable style (#4106), stabilizing the following +changes: + +- Add parentheses around `if`-`else` expressions (#2278) +- Dummy class and function implementations consisting only of `...` are formatted more + compactly (#3796) +- If an assignment statement is too long, we now prefer splitting on the right-hand side + (#3368) +- Hex codes in Unicode escape sequences are now standardized to lowercase (#2916) +- Allow empty first lines at the beginning of most blocks (#3967, #4061) +- Add parentheses around long type annotations (#3899) +- Enforce newline after module docstrings (#3932, #4028) +- Fix incorrect magic trailing comma handling in return types (#3916) +- Remove blank lines before class docstrings (#3692) +- Wrap multiple context managers in parentheses if combined in a single `with` statement + (#3489) +- Fix bug in line length calculations for power operations (#3942) +- Add trailing commas to collection literals even if there's a comment after the last + entry (#3393) +- When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) +- Add extra blank lines in stubs in a few cases (#3564, #3862) +- Accept raw strings as docstrings (#3947) +- Split long lines in case blocks (#4024) +- Stop removing spaces from walrus operators within subscripts (#3823) +- Fix incorrect formatting of certain async statements (#3609) +- Allow combining `# fmt: skip` with other comments (#3959) + +There are already a few improvements in the `--preview` style, which are slated for the +2025 stable style. Try them out and +[share your feedback](https://github.com/psf/black/issues). In the past, the preview +style has included some features that we were not able to stabilize. This year, we're +adding a separate `--unstable` style for features with known problems. Now, the +`--preview` style only includes features that we actually expect to make it into next +year's stable style. + +### Stable style + +Several bug fixes were made in features that are moved to the stable style in this +release: + +- Fix comment handling when parenthesising conditional expressions (#4134) +- Fix bug where spaces were not added around parenthesized walruses in subscripts, + unlike other binary operators (#4109) +- Remove empty lines before docstrings in async functions (#4132) +- Address a missing case in the change to allow empty lines at the beginning of all + blocks, except immediately before a docstring (#4130) +- For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) + +### Preview style + +- Add `--unstable` style, covering preview features that have known problems that would + block them from going into the stable style. Also add the `--enable-unstable-feature` + flag; for example, use + `--enable-unstable-feature hug_parens_with_braces_and_square_brackets` to apply this + preview feature throughout 2024, even if a later Black release downgrades the feature + to unstable (#4096) +- Format module docstrings the same as class and function docstrings (#4095) +- Fix crash when using a walrus in a dictionary (#4155) +- Fix unnecessary parentheses when wrapping long dicts (#4135) +- Stop normalizing spaces before `# fmt: skip` comments (#4146) + +### Configuration + +- Print warning when configuration in `pyproject.toml` contains an invalid key (#4165) +- Fix symlink handling, properly ignoring symlinks that point outside of root (#4161) +- Fix cache mtime logic that resulted in false positive cache hits (#4128) +- Remove the long-deprecated `--experimental-string-processing` flag. This feature can + currently be enabled with `--preview --enable-unstable-feature string_processing`. + (#4096) + +### Integrations + +- Revert the change to run Black's pre-commit integration only on specific git hooks + (#3940) for better compatibility with older versions of pre-commit (#4137) + +## 23.12.1 + +### Packaging + +- Fixed a bug that included dependencies from the `d` extra by default (#4108) + +## 23.12.0 + +### Highlights + +It's almost 2024, which means it's time for a new edition of _Black_'s stable style! +Together with this release, we'll put out an alpha release 24.1a1 showcasing the draft +2024 stable style, which we'll finalize in the January release. Please try it out and +[share your feedback](https://github.com/psf/black/issues/4042). + +This release (23.12.0) will still produce the 2023 style. Most but not all of the +changes in `--preview` mode will be in the 2024 stable style. + +### Stable style + +- Fix bug where `# fmt: off` automatically dedents when used with the `--line-ranges` + option, even when it is not within the specified line range. (#4084) +- Fix feature detection for parenthesized context managers (#4104) + +### Preview style + +- Prefer more equal signs before a break when splitting chained assignments (#4010) +- Standalone form feed characters at the module level are no longer removed (#4021) +- Additional cases of immediately nested tuples, lists, and dictionaries are now + indented less (#4012) +- Allow empty lines at the beginning of all blocks, except immediately before a + docstring (#4060) +- Fix crash in preview mode when using a short `--line-length` (#4086) +- Keep suites consisting of only an ellipsis on their own lines if they are not + functions or class definitions (#4066) (#4103) + +### Configuration + +- `--line-ranges` now skips _Black_'s internal stability check in `--safe` mode. This + avoids a crash on rare inputs that have many unformatted same-content lines. (#4034) + +### Packaging + +- Upgrade to mypy 1.7.1 (#4049) (#4069) +- Faster compiled wheels are now available for CPython 3.12 (#4070) + +### Integrations + +- Enable 3.12 CI (#4035) +- Build docker images in parallel (#4054) +- Build docker images with 3.12 (#4055) + +## 23.11.0 + +### Highlights + +- Support formatting ranges of lines with the new `--line-ranges` command-line option + (#4020) + +### Stable style + +- Fix crash on formatting bytes strings that look like docstrings (#4003) +- Fix crash when whitespace followed a backslash before newline in a docstring (#4008) +- Fix standalone comments inside complex blocks crashing Black (#4016) +- Fix crash on formatting code like `await (a ** b)` (#3994) +- No longer treat leading f-strings as docstrings. This matches Python's behaviour and + fixes a crash (#4019) + +### Preview style + +- Multiline dicts and lists that are the sole argument to a function are now indented + less (#3964) +- Multiline unpacked dicts and lists as the sole argument to a function are now also + indented less (#3992) +- In f-string debug expressions, quote types that are visible in the final string are + now preserved (#4005) +- Fix a bug where long `case` blocks were not split into multiple lines. Also enable + general trailing comma rules on `case` blocks (#4024) +- Keep requiring two empty lines between module-level docstring and first function or + class definition (#4028) +- Add support for single-line format skip with other comments on the same line (#3959) + +### Configuration + +- Consistently apply force exclusion logic before resolving symlinks (#4015) +- Fix a bug in the matching of absolute path names in `--include` (#3976) + +### Performance + +- Fix mypyc builds on arm64 on macOS (#4017) + +### Integrations + +- Black's pre-commit integration will now run only on git hooks appropriate for a code + formatter (#3940) + +## 23.10.1 + +### Highlights + +- Maintenance release to get a fix out for GitHub Action edge case (#3957) + +### Preview style + +- Fix merging implicit multiline strings that have inline comments (#3956) +- Allow empty first line after block open before a comment or compound statement (#3967) + +### Packaging + +- Change Dockerfile to hatch + compile black (#3965) + +### Integrations + +- The summary output for GitHub workflows is now suppressible using the `summary` + parameter. (#3958) +- Fix the action failing when Black check doesn't pass (#3957) + +### Documentation + +- It is known Windows documentation CI is broken + https://github.com/psf/black/issues/3968 + +## 23.10.0 + +### Stable style + +- Fix comments getting removed from inside parenthesized strings (#3909) + +### Preview style + +- Fix long lines with power operators getting split before the line length (#3942) +- Long type hints are now wrapped in parentheses and properly indented when split across + multiple lines (#3899) +- Magic trailing commas are now respected in return types. (#3916) +- Require one empty line after module-level docstrings. (#3932) +- Treat raw triple-quoted strings as docstrings (#3947) + +### Configuration + +- Fix cache versioning logic when `BLACK_CACHE_DIR` is set (#3937) + +### Parser + +- Fix bug where attributes named `type` were not accepted inside `match` statements + (#3950) +- Add support for PEP 695 type aliases containing lambdas and other unusual expressions + (#3949) + +### Output + +- Black no longer attempts to provide special errors for attempting to format Python 2 + code (#3933) +- Black will more consistently print stacktraces on internal errors in verbose mode + (#3938) + +### Integrations + +- The action output displayed in the job summary is now wrapped in Markdown (#3914) + +## 23.9.1 + +Due to various issues, the previous release (23.9.0) did not include compiled mypyc +wheels, which make Black significantly faster. These issues have now been fixed, and +this release should come with compiled wheels once again. + +There will be no wheels for Python 3.12 due to a bug in mypyc. We will provide 3.12 +wheels in a future release as soon as the mypyc bug is fixed. + +### Packaging + +- Upgrade to mypy 1.5.1 (#3864) + +### Performance + +- Store raw tuples instead of NamedTuples in Black's cache, improving performance and + decreasing the size of the cache (#3877) + +## 23.9.0 + +### Preview style + +- More concise formatting for dummy implementations (#3796) +- In stub files, add a blank line between a statement with a body (e.g an + `if sys.version_info > (3, x):`) and a function definition on the same level (#3862) +- Fix a bug whereby spaces were removed from walrus operators within subscript(#3823) + +### Configuration + +- Black now applies exclusion and ignore logic before resolving symlinks (#3846) + +### Performance + +- Avoid importing `IPython` if notebook cells do not contain magics (#3782) +- Improve caching by comparing file hashes as fallback for mtime and size (#3821) + +### _Blackd_ + +- Fix an issue in `blackd` with single character input (#3558) + +### Integrations + +- Black now has an + [official pre-commit mirror](https://github.com/psf/black-pre-commit-mirror). Swapping + `https://github.com/psf/black` to `https://github.com/psf/black-pre-commit-mirror` in + your `.pre-commit-config.yaml` will make Black about 2x faster (#3828) +- The `.black.env` folder specified by `ENV_PATH` will now be removed on the completion + of the GitHub Action (#3759) + +## 23.7.0 + +### Highlights + +- Runtime support for Python 3.7 has been removed. Formatting 3.7 code will still be + supported until further notice (#3765) + +### Stable style + +- Fix a bug where an illegal trailing comma was added to return type annotations using + PEP 604 unions (#3735) +- Fix several bugs and crashes where comments in stub files were removed or mishandled + under some circumstances (#3745) +- Fix a crash with multi-line magic comments like `type: ignore` within parentheses + (#3740) +- Fix error in AST validation when _Black_ removes trailing whitespace in a type comment + (#3773) + +### Preview style + +- Implicitly concatenated strings used as function args are no longer wrapped inside + parentheses (#3640) +- Remove blank lines between a class definition and its docstring (#3692) + +### Configuration + +- The `--workers` argument to _Black_ can now be specified via the `BLACK_NUM_WORKERS` + environment variable (#3743) +- `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) +- Fix _Black_ not honouring `pyproject.toml` settings when running `--stdin-filename` + and the `pyproject.toml` found isn't in the current working directory (#3719) +- _Black_ will now error if `exclude` and `extend-exclude` have invalid data types in + `pyproject.toml`, instead of silently doing the wrong thing (#3764) + +### Packaging + +- Upgrade mypyc from 0.991 to 1.3 (#3697) +- Remove patching of Click that mitigated errors on Python 3.6 with `LANG=C` (#3768) + +### Parser + +- Add support for the new PEP 695 syntax in Python 3.12 (#3703) + +### Performance + +- Speed up _Black_ significantly when the cache is full (#3751) +- Avoid importing `IPython` in a case where we wouldn't need it (#3748) + +### Output + +- Use aware UTC datetimes internally, avoids deprecation warning on Python 3.12 (#3728) +- Change verbose logging to exactly mirror _Black_'s logic for source discovery (#3749) + +### _Blackd_ + +- The `blackd` argument parser now shows the default values for options in their help + text (#3712) + +### Integrations + +- Black is now tested with + [`PYTHONWARNDEFAULTENCODING = 1`](https://docs.python.org/3/library/io.html#io-encoding-warning) + (#3763) +- Update GitHub Action to display black output in the job summary (#3688) + +### Documentation + +- Add a CITATION.cff file to the root of the repository, containing metadata on how to + cite this software (#3723) +- Update the _classes_ and _exceptions_ documentation in Developer reference to match + the latest code base (#3755) + +## 23.3.0 + +### Highlights + +This release fixes a longstanding confusing behavior in Black's GitHub action, where the +version of the action did not determine the version of Black being run (issue #3382). In +addition, there is a small bug fix around imports and a number of improvements to the +preview style. + +Please try out the +[preview style](https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html#preview-style) +with `black --preview` and tell us your feedback. All changes in the preview style are +expected to become part of Black's stable style in January 2024. + +### Stable style + +- Import lines with `# fmt: skip` and `# fmt: off` no longer have an extra blank line + added when they are right after another import line (#3610) + +### Preview style + +- Add trailing commas to collection literals even if there's a comment after the last + entry (#3393) +- `async def`, `async for`, and `async with` statements are now formatted consistently + compared to their non-async version. (#3609) +- `with` statements that contain two context managers will be consistently wrapped in + parentheses (#3589) +- Let string splitters respect [East Asian Width](https://www.unicode.org/reports/tr11/) + (#3445) +- Now long string literals can be split after East Asian commas and periods (`、` U+3001 + IDEOGRAPHIC COMMA, `。` U+3002 IDEOGRAPHIC FULL STOP, & `，` U+FF0C FULLWIDTH COMMA) + besides before spaces (#3445) +- For stubs, enforce one blank line after a nested class with a body other than just + `...` (#3564) +- Improve handling of multiline strings by changing line split behavior (#1879) + +### Parser + +- Added support for formatting files with invalid type comments (#3594) + +### Integrations + +- Update GitHub Action to use the version of Black equivalent to action's version if + version input is not specified (#3543) +- Fix missing Python binary path in autoload script for vim (#3508) + +### Documentation + +- Document that only the most recent release is supported for security issues; + vulnerabilities should be reported through Tidelift (#3612) + +## 23.1.0 + +### Highlights + +This is the first release of 2023, and following our +[stability policy](https://black.readthedocs.io/en/stable/the_black_code_style/index.html#stability-policy), +it comes with a number of improvements to our stable style, including improvements to +empty line handling, removal of redundant parentheses in several contexts, and output +that highlights implicitly concatenated strings better. + +There are also many changes to the preview style; try out `black --preview` and give us +feedback to help us set the stable style for next year. + +In addition to style changes, Black now automatically infers the supported Python +versions from your `pyproject.toml` file, removing the need to set Black's target +versions separately. + +### Stable style + +- Introduce the 2023 stable style, which incorporates most aspects of last year's + preview style (#3418). Specific changes: + - Enforce empty lines before classes and functions with sticky leading comments + (#3302) (22.12.0) + - Reformat empty and whitespace-only files as either an empty file (if no newline is + present) or as a single newline character (if a newline is present) (#3348) + (22.12.0) + - Implicitly concatenated strings used as function args are now wrapped inside + parentheses (#3307) (22.12.0) + - Correctly handle trailing commas that are inside a line's leading non-nested parens + (#3370) (22.12.0) + - `--skip-string-normalization` / `-S` now prevents docstring prefixes from being + normalized as expected (#3168) (since 22.8.0) + - When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) (22.8.0) + - Implicitly concatenated strings inside a list, set, or tuple are now wrapped inside + parentheses (#3162) (22.8.0) + - Fix a string merging/split issue when a comment is present in the middle of + implicitly concatenated strings on its own line (#3227) (22.8.0) + - Docstring quotes are no longer moved if it would violate the line length limit + (#3044, #3430) (22.6.0) + - Parentheses around return annotations are now managed (#2990) (22.6.0) + - Remove unnecessary parentheses around awaited objects (#2991) (22.6.0) + - Remove unnecessary parentheses in `with` statements (#2926) (22.6.0) + - Remove trailing newlines after code block open (#3035) (22.6.0) + - Code cell separators `#%%` are now standardised to `# %%` (#2919) (22.3.0) + - Remove unnecessary parentheses from `except` statements (#2939) (22.3.0) + - Remove unnecessary parentheses from tuple unpacking in `for` loops (#2945) (22.3.0) + - Avoid magic-trailing-comma in single-element subscripts (#2942) (22.3.0) +- Fix a crash when a colon line is marked between `# fmt: off` and `# fmt: on` (#3439) + +### Preview style + +- Format hex codes in unicode escape sequences in string literals (#2916) +- Add parentheses around `if`-`else` expressions (#2278) +- Improve performance on large expressions that contain many strings (#3467) +- Fix a crash in preview style with assert + parenthesized string (#3415) +- Fix crashes in preview style with walrus operators used in function return annotations + and except clauses (#3423) +- Fix a crash in preview advanced string processing where mixed implicitly concatenated + regular and f-strings start with an empty span (#3463) +- Fix a crash in preview advanced string processing where a standalone comment is placed + before a dict's value (#3469) +- Fix an issue where extra empty lines are added when a decorator has `# fmt: skip` + applied or there is a standalone comment between decorators (#3470) +- Do not put the closing quotes in a docstring on a separate line, even if the line is + too long (#3430) +- Long values in dict literals are now wrapped in parentheses; correspondingly + unnecessary parentheses around short values in dict literals are now removed; long + string lambda values are now wrapped in parentheses (#3440) +- Fix two crashes in preview style involving edge cases with docstrings (#3451) +- Exclude string type annotations from improved string processing; fix crash when the + return type annotation is stringified and spans across multiple lines (#3462) +- Wrap multiple context managers in parentheses when targeting Python 3.9+ (#3489) +- Fix several crashes in preview style with walrus operators used in `with` statements + or tuples (#3473) +- Fix an invalid quote escaping bug in f-string expressions where it produced invalid + code. Implicitly concatenated f-strings with different quotes can now be merged or + quote-normalized by changing the quotes used in expressions. (#3509) +- Fix crash on `await (yield)` when Black is compiled with mypyc (#3533) + +### Configuration + +- Black now tries to infer its `--target-version` from the project metadata specified in + `pyproject.toml` (#3219) + +### Packaging + +- Upgrade mypyc from `0.971` to `0.991` so mypycified _Black_ can be built on armv7 + (#3380) + - This also fixes some crashes while using compiled Black with a debug build of + CPython +- Drop specific support for the `tomli` requirement on 3.11 alpha releases, working + around a bug that would cause the requirement not to be installed on any non-final + Python releases (#3448) +- Black now depends on `packaging` version `22.0` or later. This is required for new + functionality that needs to parse part of the project metadata (#3219) + +### Output + +- Calling `black --help` multiple times will return the same help contents each time + (#3516) +- Verbose logging now shows the values of `pyproject.toml` configuration variables + (#3392) +- Fix false symlink detection messages in verbose output due to using an incorrect + relative path to the project root (#3385) + +### Integrations + +- Move 3.11 CI to normal flow now that all dependencies support 3.11 (#3446) +- Docker: Add new `latest_prerelease` tag automation to follow latest black alpha + release on docker images (#3465) + +### Documentation + +- Expand `vim-plug` installation instructions to offer more explicit options (#3468) + +## 22.12.0 + +### Preview style + +- Enforce empty lines before classes and functions with sticky leading comments (#3302) +- Reformat empty and whitespace-only files as either an empty file (if no newline is + present) or as a single newline character (if a newline is present) (#3348) +- Implicitly concatenated strings used as function args are now wrapped inside + parentheses (#3307) +- For assignment statements, prefer splitting the right hand side if the left hand side + fits on a single line (#3368) +- Correctly handle trailing commas that are inside a line's leading non-nested parens + (#3370) + +### Configuration + +- Fix incorrectly applied `.gitignore` rules by considering the `.gitignore` location + and the relative path to the target file (#3338) +- Fix incorrectly ignoring `.gitignore` presence when more than one source directory is + specified (#3336) + +### Parser + +- Parsing support has been added for walruses inside generator expression that are + passed as function args (for example, + `any(match := my_re.match(text) for text in texts)`) (#3327). + +### Integrations + +- Vim plugin: Optionally allow using the system installation of Black via + `let g:black_use_virtualenv = 0`(#3309) + +## 22.10.0 + +### Highlights + +- Runtime support for Python 3.6 has been removed. Formatting 3.6 code will still be + supported until further notice. + +### Stable style + +- Fix a crash when `# fmt: on` is used on a different block level than `# fmt: off` + (#3281) + +### Preview style + +- Fix a crash when formatting some dicts with parenthesis-wrapped long string keys + (#3262) + +### Configuration + +- `.ipynb_checkpoints` directories are now excluded by default (#3293) +- Add `--skip-source-first-line` / `-x` option to ignore the first line of source code + while formatting (#3299) + +### Packaging + +- Executables made with PyInstaller will no longer crash when formatting several files + at once on macOS. Native x86-64 executables for macOS are available once again. + (#3275) +- Hatchling is now used as the build backend. This will not have any effect for users + who install Black with its wheels from PyPI. (#3233) +- Faster compiled wheels are now available for CPython 3.11 (#3276) + +### _Blackd_ + +- Windows style (CRLF) newlines will be preserved (#3257). + +### Integrations + +- Vim plugin: add flag (`g:black_preview`) to enable/disable the preview style (#3246) +- Update GitHub Action to support formatting of Jupyter Notebook files via a `jupyter` + option (#3282) +- Update GitHub Action to support use of version specifiers (e.g. `<23`) for Black + version (#3265) + +## 22.8.0 + +### Highlights + +- Python 3.11 is now supported, except for _blackd_ as aiohttp does not support 3.11 as + of publishing (#3234) +- This is the last release that supports running _Black_ on Python 3.6 (formatting 3.6 + code will continue to be supported until further notice) +- Reword the stability policy to say that we may, in rare cases, make changes that + affect code that was not previously formatted by _Black_ (#3155) + +### Stable style + +- Fix an infinite loop when using `# fmt: on/off` in the middle of an expression or code + block (#3158) +- Fix incorrect handling of `# fmt: skip` on colon (`:`) lines (#3148) +- Comments are no longer deleted when a line had spaces removed around power operators + (#2874) + +### Preview style + +- Single-character closing docstring quotes are no longer moved to their own line as + this is invalid. This was a bug introduced in version 22.6.0. (#3166) +- `--skip-string-normalization` / `-S` now prevents docstring prefixes from being + normalized as expected (#3168) +- When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) +- Implicitly concatenated strings inside a list, set, or tuple are now wrapped inside + parentheses (#3162) +- Fix a string merging/split issue when a comment is present in the middle of implicitly + concatenated strings on its own line (#3227) + +### _Blackd_ + +- `blackd` now supports enabling the preview style via the `X-Preview` header (#3217) + +### Configuration + +- Black now uses the presence of debug f-strings to detect target version (#3215) +- Fix misdetection of project root and verbose logging of sources in cases involving + `--stdin-filename` (#3216) +- Immediate `.gitignore` files in source directories given on the command line are now + also respected, previously only `.gitignore` files in the project root and + automatically discovered directories were respected (#3237) + +### Documentation + +- Recommend using BlackConnect in IntelliJ IDEs (#3150) + +### Integrations + +- Vim plugin: prefix messages with `Black: ` so it's clear they come from Black (#3194) +- Docker: changed to a /opt/venv installation + added to PATH to be available to + non-root users (#3202) + +### Output + +- Change from deprecated `asyncio.get_event_loop()` to create our event loop which + removes DeprecationWarning (#3164) +- Remove logging from internal `blib2to3` library since it regularly emits error logs + about failed caching that can and should be ignored (#3193) + +### Parser + +- Type comments are now included in the AST equivalence check consistently so accidental + deletion raises an error. Though type comments can't be tracked when running on PyPy + 3.7 due to standard library limitations. (#2874) + +### Performance + +- Reduce Black's startup time when formatting a single file by 15-30% (#3211) + +## 22.6.0 + +### Style + +- Fix unstable formatting involving `#fmt: skip` and `# fmt:skip` comments (notice the + lack of spaces) (#2970) + +### Preview style + +- Docstring quotes are no longer moved if it would violate the line length limit (#3044) +- Parentheses around return annotations are now managed (#2990) +- Remove unnecessary parentheses around awaited objects (#2991) +- Remove unnecessary parentheses in `with` statements (#2926) +- Remove trailing newlines after code block open (#3035) + +### Integrations + +- Add `scripts/migrate-black.py` script to ease introduction of Black to a Git project + (#3038) + +### Output + +- Output Python version and implementation as part of `--version` flag (#2997) + +### Packaging + +- Use `tomli` instead of `tomllib` on Python 3.11 builds where `tomllib` is not + available (#2987) + +### Parser + +- [PEP 654](https://peps.python.org/pep-0654/#except) syntax (for example, + `except *ExceptionGroup:`) is now supported (#3016) +- [PEP 646](https://peps.python.org/pep-0646) syntax (for example, + `Array[Batch, *Shape]` or `def fn(*args: *T) -> None`) is now supported (#3071) + +### Vim Plugin + +- Fix `strtobool` function. It didn't parse true/on/false/off. (#3025) + +## 22.3.0 + +### Preview style + +- Code cell separators `#%%` are now standardised to `# %%` (#2919) +- Remove unnecessary parentheses from `except` statements (#2939) +- Remove unnecessary parentheses from tuple unpacking in `for` loops (#2945) +- Avoid magic-trailing-comma in single-element subscripts (#2942) + +### Configuration + +- Do not format `__pypackages__` directories by default (#2836) +- Add support for specifying stable version with `--required-version` (#2832). +- Avoid crashing when the user has no homedir (#2814) +- Avoid crashing when md5 is not available (#2905) +- Fix handling of directory junctions on Windows (#2904) + +### Documentation + +- Update pylint config documentation (#2931) + +### Integrations + +- Move test to disable plugin in Vim/Neovim, which speeds up loading (#2896) + +### Output + +- In verbose mode, log when _Black_ is using user-level config (#2861) + +### Packaging + +- Fix Black to work with Click 8.1.0 (#2966) +- On Python 3.11 and newer, use the standard library's `tomllib` instead of `tomli` + (#2903) +- `black-primer`, the deprecated internal devtool, has been removed and copied to a + [separate repository](https://github.com/cooperlees/black-primer) (#2924) + +### Parser + +- Black can now parse starred expressions in the target of `for` and `async for` + statements, e.g `for item in *items_1, *items_2: pass` (#2879). + +## 22.1.0 + +At long last, _Black_ is no longer a beta product! This is the first non-beta release +and the first release covered by our new +[stability policy](https://black.readthedocs.io/en/stable/the_black_code_style/index.html#stability-policy). + +### Highlights + +- **Remove Python 2 support** (#2740) +- Introduce the `--preview` flag (#2752) + +### Style + +- Deprecate `--experimental-string-processing` and move the functionality under + `--preview` (#2789) +- For stubs, one blank line between class attributes and methods is now kept if there's + at least one pre-existing blank line (#2736) +- Black now normalizes string prefix order (#2297) +- Remove spaces around power operators if both operands are simple (#2726) +- Work around bug that causes unstable formatting in some cases in the presence of the + magic trailing comma (#2807) +- Use parentheses for attribute access on decimal float and int literals (#2799) +- Don't add whitespace for attribute access on hexadecimal, binary, octal, and complex + literals (#2799) +- Treat blank lines in stubs the same inside top-level `if` statements (#2820) +- Fix unstable formatting with semicolons and arithmetic expressions (#2817) +- Fix unstable formatting around magic trailing comma (#2572) + +### Parser + +- Fix mapping cases that contain as-expressions, like `case {"key": 1 | 2 as password}` + (#2686) +- Fix cases that contain multiple top-level as-expressions, like `case 1 as a, 2 as b` + (#2716) +- Fix call patterns that contain as-expressions with keyword arguments, like + `case Foo(bar=baz as quux)` (#2749) +- Tuple unpacking on `return` and `yield` constructs now implies 3.8+ (#2700) +- Unparenthesized tuples on annotated assignments (e.g + `values: Tuple[int, ...] = 1, 2, 3`) now implies 3.8+ (#2708) +- Fix handling of standalone `match()` or `case()` when there is a trailing newline or a + comment inside of the parentheses. (#2760) +- `from __future__ import annotations` statement now implies Python 3.7+ (#2690) + +### Performance + +- Speed-up the new backtracking parser about 4X in general (enabled when + `--target-version` is set to 3.10 and higher). (#2728) +- _Black_ is now compiled with [mypyc](https://github.com/mypyc/mypyc) for an overall 2x + speed-up. 64-bit Windows, MacOS, and Linux (not including musl) are supported. (#1009, + #2431) + +### Configuration + +- Do not accept bare carriage return line endings in pyproject.toml (#2408) +- Add configuration option (`python-cell-magics`) to format cells with custom magics in + Jupyter Notebooks (#2744) +- Allow setting custom cache directory on all platforms with environment variable + `BLACK_CACHE_DIR` (#2739). +- Enable Python 3.10+ by default, without any extra need to specify + `--target-version=py310`. (#2758) +- Make passing `SRC` or `--code` mandatory and mutually exclusive (#2804) + +### Output + +- Improve error message for invalid regular expression (#2678) +- Improve error message when parsing fails during AST safety check by embedding the + underlying SyntaxError (#2693) +- No longer color diff headers white as it's unreadable in light themed terminals + (#2691) +- Text coloring added in the final statistics (#2712) +- Verbose mode also now describes how a project root was discovered and which paths will + be formatted. (#2526) + +### Packaging + +- All upper version bounds on dependencies have been removed (#2718) +- `typing-extensions` is no longer a required dependency in Python 3.10+ (#2772) +- Set `click` lower bound to `8.0.0` (#2791) + +### Integrations + +- Update GitHub action to support containerized runs (#2748) + +### Documentation + +- Change protocol in pip installation instructions to `https://` (#2761) +- Change HTML theme to Furo primarily for its responsive design and mobile support + (#2793) +- Deprecate the `black-primer` tool (#2809) +- Document Python support policy (#2819) + +## 21.12b0 + +### _Black_ + +- Fix determination of f-string expression spans (#2654) +- Fix bad formatting of error messages about EOF in multi-line statements (#2343) +- Functions and classes in blocks now have more consistent surrounding spacing (#2472) + +#### Jupyter Notebook support + +- Cell magics are now only processed if they are known Python cell magics. Earlier, all + cell magics were tokenized, leading to possible indentation errors e.g. with + `%%writefile`. (#2630) +- Fix assignment to environment variables in Jupyter Notebooks (#2642) + +#### Python 3.10 support + +- Point users to using `--target-version py310` if we detect 3.10-only syntax (#2668) +- Fix `match` statements with open sequence subjects, like `match a, b:` or + `match a, *b:` (#2639) (#2659) +- Fix `match`/`case` statements that contain `match`/`case` soft keywords multiple + times, like `match re.match()` (#2661) +- Fix `case` statements with an inline body (#2665) +- Fix styling of starred expressions inside `match` subject (#2667) +- Fix parser error location on invalid syntax in a `match` statement (#2649) +- Fix Python 3.10 support on platforms without ProcessPoolExecutor (#2631) +- Improve parsing performance on code that uses `match` under `--target-version py310` + up to ~50% (#2670) + +### Packaging + +- Remove dependency on `regex` (#2644) (#2663) + +## 21.11b1 + +### _Black_ + +- Bumped regex version minimum to 2021.4.4 to fix Pattern class usage (#2621) + +## 21.11b0 + +### _Black_ + +- Warn about Python 2 deprecation in more cases by improving Python 2 only syntax + detection (#2592) +- Add experimental PyPy support (#2559) +- Add partial support for the match statement. As it's experimental, it's only enabled + when `--target-version py310` is explicitly specified (#2586) +- Add support for parenthesized with (#2586) +- Declare support for Python 3.10 for running Black (#2562) + +### Integrations + +- Fixed vim plugin with Python 3.10 by removing deprecated distutils import (#2610) +- The vim plugin now parses `skip_magic_trailing_comma` from pyproject.toml (#2613) + +## 21.10b0 + +### _Black_ + +- Document stability policy, that will apply for non-beta releases (#2529) +- Add new `--workers` parameter (#2514) +- Fixed feature detection for positional-only arguments in lambdas (#2532) +- Bumped typed-ast version minimum to 1.4.3 for 3.10 compatibility (#2519) +- Fixed a Python 3.10 compatibility issue where the loop argument was still being passed + even though it has been removed (#2580) +- Deprecate Python 2 formatting support (#2523) + +### _Blackd_ + +- Remove dependency on aiohttp-cors (#2500) +- Bump required aiohttp version to 3.7.4 (#2509) + +### _Black-Primer_ + +- Add primer support for --projects (#2555) +- Print primer summary after individual failures (#2570) + +### Integrations + +- Allow to pass `target_version` in the vim plugin (#1319) +- Install build tools in docker file and use multi-stage build to keep the image size + down (#2582) + +## 21.9b0 + +### Packaging + +- Fix missing modules in self-contained binaries (#2466) +- Fix missing toml extra used during installation (#2475) + +## 21.8b0 + +### _Black_ + +- Add support for formatting Jupyter Notebook files (#2357) +- Move from `appdirs` dependency to `platformdirs` (#2375) +- Present a more user-friendly error if .gitignore is invalid (#2414) +- The failsafe for accidentally added backslashes in f-string expressions has been + hardened to handle more edge cases during quote normalization (#2437) +- Avoid changing a function return type annotation's type to a tuple by adding a + trailing comma (#2384) +- Parsing support has been added for unparenthesized walruses in set literals, set + comprehensions, and indices (#2447). +- Pin `setuptools-scm` build-time dependency version (#2457) +- Exclude typing-extensions version 3.10.0.1 due to it being broken on Python 3.10 + (#2460) + +### _Blackd_ + +- Replace sys.exit(-1) with raise ImportError as it plays more nicely with tools that + scan installed packages (#2440) + +### Integrations + +- The provided pre-commit hooks no longer specify `language_version` to avoid overriding + `default_language_version` (#2430) + +## 21.7b0 + +### _Black_ + +- Configuration files using TOML features higher than spec v0.5.0 are now supported + (#2301) +- Add primer support and test for code piped into black via STDIN (#2315) +- Fix internal error when `FORCE_OPTIONAL_PARENTHESES` feature is enabled (#2332) +- Accept empty stdin (#2346) +- Provide a more useful error when parsing fails during AST safety checks (#2304) + +### Docker + +- Add new `latest_release` tag automation to follow latest black release on docker + images (#2374) + +### Integrations + +- The vim plugin now searches upwards from the directory containing the current buffer + instead of the current working directory for pyproject.toml. (#1871) +- The vim plugin now reads the correct string normalization option in pyproject.toml + (#1869) +- The vim plugin no longer crashes Black when there's boolean values in pyproject.toml + (#1869) + +## 21.6b0 + +### _Black_ + +- Fix failure caused by `fmt: skip` and indentation (#2281) +- Account for += assignment when deciding whether to split string (#2312) +- Correct max string length calculation when there are string operators (#2292) +- Fixed option usage when using the `--code` flag (#2259) +- Do not call `uvloop.install()` when _Black_ is used as a library (#2303) +- Added `--required-version` option to require a specific version to be running (#2300) +- Fix incorrect custom breakpoint indices when string group contains fake f-strings + (#2311) +- Fix regression where `R` prefixes would be lowercased for docstrings (#2285) +- Fix handling of named escapes (`\N{...}`) when `--experimental-string-processing` is + used (#2319) + +### Integrations + +- The official Black action now supports choosing what version to use, and supports the + major 3 OSes. (#1940) + +## 21.5b2 + +### _Black_ + +- A space is no longer inserted into empty docstrings (#2249) +- Fix handling of .gitignore files containing non-ASCII characters on Windows (#2229) +- Respect `.gitignore` files in all levels, not only `root/.gitignore` file (apply + `.gitignore` rules like `git` does) (#2225) +- Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) +- Add extra uvloop install + import support if in python env (#2258) +- Fix --experimental-string-processing crash when matching parens are not found (#2283) +- Make sure to split lines that start with a string operator (#2286) +- Fix regular expression that black uses to identify f-expressions (#2287) + +### _Blackd_ + +- Add a lower bound for the `aiohttp-cors` dependency. Only 0.4.0 or higher is + supported. (#2231) + +### Packaging + +- Release self-contained x86_64 MacOS binaries as part of the GitHub release pipeline + (#2198) +- Always build binaries with the latest available Python (#2260) + +### Documentation + +- Add discussion of magic comments to FAQ page (#2272) +- `--experimental-string-processing` will be enabled by default in the future (#2273) +- Fix typos discovered by codespell (#2228) +- Fix Vim plugin installation instructions. (#2235) +- Add new Frequently Asked Questions page (#2247) +- Fix encoding + symlink issues preventing proper build on Windows (#2262) + +## 21.5b1 + +### _Black_ + +- Refactor `src/black/__init__.py` into many files (#2206) + +### Documentation + +- Replaced all remaining references to the + [`master`](https://github.com/psf/black/tree/main) branch with the + [`main`](https://github.com/psf/black/tree/main) branch. Some additional changes in + the source code were also made. (#2210) +- Significantly reorganized the documentation to make much more sense. Check them out by + heading over to [the stable docs on RTD](https://black.readthedocs.io/en/stable/). + (#2174) + +## 21.5b0 + +### _Black_ + +- Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) +- Stop detecting target version as Python 3.9+ with pre-PEP-614 decorators that are + being called but with no arguments (#2182) + +### _Black-Primer_ + +- Add `--no-diff` to black-primer to suppress formatting changes (#2187) + +## 21.4b2 + +### _Black_ + +- Fix crash if the user configuration directory is inaccessible. (#2158) + +- Clarify + [circumstances](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) + in which _Black_ may change the AST (#2159) + +- Allow `.gitignore` rules to be overridden by specifying `exclude` in `pyproject.toml` + or on the command line. (#2170) + +### _Packaging_ + +- Install `primer.json` (used by `black-primer` by default) with black. (#2154) + +## 21.4b1 + +### _Black_ + +- Fix crash on docstrings ending with "\\ ". (#2142) + +- Fix crash when atypical whitespace is cleaned out of dostrings (#2120) + +- Reflect the `--skip-magic-trailing-comma` and `--experimental-string-processing` flags + in the name of the cache file. Without this fix, changes in these flags would not take + effect if the cache had already been populated. (#2131) + +- Don't remove necessary parentheses from assignment expression containing assert / + return statements. (#2143) + +### _Packaging_ + +- Bump pathspec to >= 0.8.1 to solve invalid .gitignore exclusion handling + +## 21.4b0 + +### _Black_ + +- Fixed a rare but annoying formatting instability created by the combination of + optional trailing commas inserted by `Black` and optional parentheses looking at + pre-existing "magic" trailing commas. This fixes issue #1629 and all of its many many + duplicates. (#2126) + +- `Black` now processes one-line docstrings by stripping leading and trailing spaces, + and adding a padding space when needed to break up """". (#1740) + +- `Black` now cleans up leading non-breaking spaces in comments (#2092) + +- `Black` now respects `--skip-string-normalization` when normalizing multiline + docstring quotes (#1637) + +- `Black` no longer removes all empty lines between non-function code and decorators + when formatting typing stubs. Now `Black` enforces a single empty line. (#1646) + +- `Black` no longer adds an incorrect space after a parenthesized assignment expression + in if/while statements (#1655) + +- Added `--skip-magic-trailing-comma` / `-C` to avoid using trailing commas as a reason + to split lines (#1824) + +- fixed a crash when PWD=/ on POSIX (#1631) + +- fixed "I/O operation on closed file" when using --diff (#1664) + +- Prevent coloured diff output being interleaved with multiple files (#1673) + +- Added support for PEP 614 relaxed decorator syntax on python 3.9 (#1711) + +- Added parsing support for unparenthesized tuples and yield expressions in annotated + assignments (#1835) + +- added `--extend-exclude` argument (PR #2005) + +- speed up caching by avoiding pathlib (#1950) + +- `--diff` correctly indicates when a file doesn't end in a newline (#1662) + +- Added `--stdin-filename` argument to allow stdin to respect `--force-exclude` rules + (#1780) + +- Lines ending with `fmt: skip` will now be not formatted (#1800) + +- PR #2053: Black no longer relies on typed-ast for Python 3.8 and higher + +- PR #2053: Python 2 support is now optional, install with + `python3 -m pip install black[python2]` to maintain support. + +- Exclude `venv` directory by default (#1683) + +- Fixed "Black produced code that is not equivalent to the source" when formatting + Python 2 docstrings (#2037) + +### _Packaging_ + +- Self-contained native _Black_ binaries are now provided for releases via GitHub + Releases (#1743) + +## 20.8b1 + +### _Packaging_ + +- explicitly depend on Click 7.1.2 or newer as `Black` no longer works with versions + older than 7.0 + +## 20.8b0 + +### _Black_ + +- re-implemented support for explicit trailing commas: now it works consistently within + any bracket pair, including nested structures (#1288 and duplicates) + +- `Black` now reindents docstrings when reindenting code around it (#1053) + +- `Black` now shows colored diffs (#1266) + +- `Black` is now packaged using 'py3' tagged wheels (#1388) + +- `Black` now supports Python 3.8 code, e.g. star expressions in return statements + (#1121) + +- `Black` no longer normalizes capital R-string prefixes as those have a + community-accepted meaning (#1244) + +- `Black` now uses exit code 2 when specified configuration file doesn't exit (#1361) + +- `Black` now works on AWS Lambda (#1141) + +- added `--force-exclude` argument (#1032) + +- removed deprecated `--py36` option (#1236) + +- fixed `--diff` output when EOF is encountered (#526) + +- fixed `# fmt: off` handling around decorators (#560) + +- fixed unstable formatting with some `# type: ignore` comments (#1113) + +- fixed invalid removal on organizing brackets followed by indexing (#1575) + +- introduced `black-primer`, a CI tool that allows us to run regression tests against + existing open source users of Black (#1402) + +- introduced property-based fuzzing to our test suite based on Hypothesis and + Hypothersmith (#1566) + +- implemented experimental and disabled by default long string rewrapping (#1132), + hidden under a `--experimental-string-processing` flag while it's being worked on; + this is an undocumented and unsupported feature, you lose Internet points for + depending on it (#1609) + +### Vim plugin + +- prefer virtualenv packages over global packages (#1383) + +## 19.10b0 + +- added support for PEP 572 assignment expressions (#711) + +- added support for PEP 570 positional-only arguments (#943) + +- added support for async generators (#593) + +- added support for pre-splitting collections by putting an explicit trailing comma + inside (#826) + +- added `black -c` as a way to format code passed from the command line (#761) + +- --safe now works with Python 2 code (#840) + +- fixed grammar selection for Python 2-specific code (#765) + +- fixed feature detection for trailing commas in function definitions and call sites + (#763) + +- `# fmt: off`/`# fmt: on` comment pairs placed multiple times within the same block of + code now behave correctly (#1005) + +- _Black_ no longer crashes on Windows machines with more than 61 cores (#838) + +- _Black_ no longer crashes on standalone comments prepended with a backslash (#767) + +- _Black_ no longer crashes on `from` ... `import` blocks with comments (#829) + +- _Black_ no longer crashes on Python 3.7 on some platform configurations (#494) + +- _Black_ no longer fails on comments in from-imports (#671) + +- _Black_ no longer fails when the file starts with a backslash (#922) + +- _Black_ no longer merges regular comments with type comments (#1027) + +- _Black_ no longer splits long lines that contain type comments (#997) + +- removed unnecessary parentheses around `yield` expressions (#834) + +- added parentheses around long tuples in unpacking assignments (#832) + +- added parentheses around complex powers when they are prefixed by a unary operator + (#646) + +- fixed bug that led _Black_ format some code with a line length target of 1 (#762) + +- _Black_ no longer introduces quotes in f-string subexpressions on string boundaries + (#863) + +- if _Black_ puts parenthesis around a single expression, it moves comments to the + wrapped expression instead of after the brackets (#872) + +- `blackd` now returns the version of _Black_ in the response headers (#1013) + +- `blackd` can now output the diff of formats on source code when the `X-Diff` header is + provided (#969) + +## 19.3b0 + +- new option `--target-version` to control which Python versions _Black_-formatted code + should target (#618) + +- deprecated `--py36` (use `--target-version=py36` instead) (#724) + +- _Black_ no longer normalizes numeric literals to include `_` separators (#696) + +- long `del` statements are now split into multiple lines (#698) + +- type comments are no longer mangled in function signatures + +- improved performance of formatting deeply nested data structures (#509) + +- _Black_ now properly formats multiple files in parallel on Windows (#632) + +- _Black_ now creates cache files atomically which allows it to be used in parallel + pipelines (like `xargs -P8`) (#673) + +- _Black_ now correctly indents comments in files that were previously formatted with + tabs (#262) + +- `blackd` now supports CORS (#622) + +## 18.9b0 + +- numeric literals are now formatted by _Black_ (#452, #461, #464, #469): + + - numeric literals are normalized to include `_` separators on Python 3.6+ code + + - added `--skip-numeric-underscore-normalization` to disable the above behavior and + leave numeric underscores as they were in the input + + - code with `_` in numeric literals is recognized as Python 3.6+ + + - most letters in numeric literals are lowercased (e.g., in `1e10`, `0x01`) + + - hexadecimal digits are always uppercased (e.g. `0xBADC0DE`) + +- added `blackd`, see + [its documentation](https://github.com/psf/black/blob/18.9b0/README.md#blackd) for + more info (#349) + +- adjacent string literals are now correctly split into multiple lines (#463) + +- trailing comma is now added to single imports that don't fit on a line (#250) + +- cache is now populated when `--check` is successful for a file which speeds up + consecutive checks of properly formatted unmodified files (#448) + +- whitespace at the beginning of the file is now removed (#399) + +- fixed mangling [pweave](http://mpastell.com/pweave/) and + [Spyder IDE](https://www.spyder-ide.org/) special comments (#532) + +- fixed unstable formatting when unpacking big tuples (#267) + +- fixed parsing of `__future__` imports with renames (#389) + +- fixed scope of `# fmt: off` when directly preceding `yield` and other nodes (#385) + +- fixed formatting of lambda expressions with default arguments (#468) + +- fixed `async for` statements: _Black_ no longer breaks them into separate lines (#372) + +- note: the Vim plugin stopped registering `,=` as a default chord as it turned out to + be a bad idea (#415) + +## 18.6b4 + +- hotfix: don't freeze when multiple comments directly precede `# fmt: off` (#371) + +## 18.6b3 + +- typing stub files (`.pyi`) now have blank lines added after constants (#340) + +- `# fmt: off` and `# fmt: on` are now much more dependable: + + - they now work also within bracket pairs (#329) + + - they now correctly work across function/class boundaries (#335) + + - they now work when an indentation block starts with empty lines or misaligned + comments (#334) + +- made Click not fail on invalid environments; note that Click is right but the + likelihood we'll need to access non-ASCII file paths when dealing with Python source + code is low (#277) + +- fixed improper formatting of f-strings with quotes inside interpolated expressions + (#322) + +- fixed unnecessary slowdown when long list literals where found in a file + +- fixed unnecessary slowdown on AST nodes with very many siblings + +- fixed cannibalizing backslashes during string normalization + +- fixed a crash due to symbolic links pointing outside of the project directory (#338) + +## 18.6b2 + +- added `--config` (#65) + +- added `-h` equivalent to `--help` (#316) + +- fixed improper unmodified file caching when `-S` was used + +- fixed extra space in string unpacking (#305) + +- fixed formatting of empty triple quoted strings (#313) + +- fixed unnecessary slowdown in comment placement calculation on lines without comments + +## 18.6b1 + +- hotfix: don't output human-facing information on stdout (#299) + +- hotfix: don't output cake emoji on non-zero return code (#300) + +## 18.6b0 + +- added `--include` and `--exclude` (#270) + +- added `--skip-string-normalization` (#118) + +- added `--verbose` (#283) + +- the header output in `--diff` now actually conforms to the unified diff spec + +- fixed long trivial assignments being wrapped in unnecessary parentheses (#273) + +- fixed unnecessary parentheses when a line contained multiline strings (#232) + +- fixed stdin handling not working correctly if an old version of Click was used (#276) + +- _Black_ now preserves line endings when formatting a file in place (#258) + +## 18.5b1 + +- added `--pyi` (#249) + +- added `--py36` (#249) + +- Python grammar pickle caches are stored with the formatting caches, making _Black_ + work in environments where site-packages is not user-writable (#192) + +- _Black_ now enforces a PEP 257 empty line after a class-level docstring (and/or + fields) and the first method + +- fixed invalid code produced when standalone comments were present in a trailer that + was omitted from line splitting on a large expression (#237) + +- fixed optional parentheses being removed within `# fmt: off` sections (#224) + +- fixed invalid code produced when stars in very long imports were incorrectly wrapped + in optional parentheses (#234) + +- fixed unstable formatting when inline comments were moved around in a trailer that was + omitted from line splitting on a large expression (#238) + +- fixed extra empty line between a class declaration and the first method if no class + docstring or fields are present (#219) + +- fixed extra empty line between a function signature and an inner function or inner + class (#196) + +## 18.5b0 + +- call chains are now formatted according to the + [fluent interfaces](https://en.wikipedia.org/wiki/Fluent_interface) style (#67) + +- data structure literals (tuples, lists, dictionaries, and sets) are now also always + exploded like imports when they don't fit in a single line (#152) + +- slices are now formatted according to PEP 8 (#178) + +- parentheses are now also managed automatically on the right-hand side of assignments + and return statements (#140) + +- math operators now use their respective priorities for delimiting multiline + expressions (#148) + +- optional parentheses are now omitted on expressions that start or end with a bracket + and only contain a single operator (#177) + +- empty parentheses in a class definition are now removed (#145, #180) + +- string prefixes are now standardized to lowercase and `u` is removed on Python 3.6+ + only code and Python 2.7+ code with the `unicode_literals` future import (#188, #198, + #199) + +- typing stub files (`.pyi`) are now formatted in a style that is consistent with PEP + 484 (#207, #210) + +- progress when reformatting many files is now reported incrementally + +- fixed trailers (content with brackets) being unnecessarily exploded into their own + lines after a dedented closing bracket (#119) + +- fixed an invalid trailing comma sometimes left in imports (#185) + +- fixed non-deterministic formatting when multiple pairs of removable parentheses were + used (#183) + +- fixed multiline strings being unnecessarily wrapped in optional parentheses in long + assignments (#215) + +- fixed not splitting long from-imports with only a single name + +- fixed Python 3.6+ file discovery by also looking at function calls with unpacking. + This fixed non-deterministic formatting if trailing commas where used both in function + signatures with stars and function calls with stars but the former would be + reformatted to a single line. + +- fixed crash on dealing with optional parentheses (#193) + +- fixed "is", "is not", "in", and "not in" not considered operators for splitting + purposes + +- fixed crash when dead symlinks where encountered + +## 18.4a4 + +- don't populate the cache on `--check` (#175) + +## 18.4a3 + +- added a "cache"; files already reformatted that haven't changed on disk won't be + reformatted again (#109) + +- `--check` and `--diff` are no longer mutually exclusive (#149) + +- generalized star expression handling, including double stars; this fixes + multiplication making expressions "unsafe" for trailing commas (#132) + +- _Black_ no longer enforces putting empty lines behind control flow statements (#90) + +- _Black_ now splits imports like "Mode 3 + trailing comma" of isort (#127) + +- fixed comment indentation when a standalone comment closes a block (#16, #32) + +- fixed standalone comments receiving extra empty lines if immediately preceding a + class, def, or decorator (#56, #154) + +- fixed `--diff` not showing entire path (#130) + +- fixed parsing of complex expressions after star and double stars in function calls + (#2) + +- fixed invalid splitting on comma in lambda arguments (#133) + +- fixed missing splits of ternary expressions (#141) + +## 18.4a2 + +- fixed parsing of unaligned standalone comments (#99, #112) + +- fixed placement of dictionary unpacking inside dictionary literals (#111) + +- Vim plugin now works on Windows, too + +- fixed unstable formatting when encountering unnecessarily escaped quotes in a string + (#120) + +## 18.4a1 + +- added `--quiet` (#78) + +- added automatic parentheses management (#4) + +- added [pre-commit](https://pre-commit.com) integration (#103, #104) + +- fixed reporting on `--check` with multiple files (#101, #102) + +- fixed removing backslash escapes from raw strings (#100, #105) + +## 18.4a0 + +- added `--diff` (#87) + +- add line breaks before all delimiters, except in cases like commas, to better comply + with PEP 8 (#73) + +- standardize string literals to use double quotes (almost) everywhere (#75) + +- fixed handling of standalone comments within nested bracketed expressions; _Black_ + will no longer produce super long lines or put all standalone comments at the end of + the expression (#22) + +- fixed 18.3a4 regression: don't crash and burn on empty lines with trailing whitespace + (#80) + +- fixed 18.3a4 regression: `# yapf: disable` usage as trailing comment would cause + _Black_ to not emit the rest of the file (#95) + +- when CTRL+C is pressed while formatting many files, _Black_ no longer freaks out with + a flurry of asyncio-related exceptions + +- only allow up to two empty lines on module level and only single empty lines within + functions (#74) + +## 18.3a4 + +- `# fmt: off` and `# fmt: on` are implemented (#5) + +- automatic detection of deprecated Python 2 forms of print statements and exec + statements in the formatted file (#49) + +- use proper spaces for complex expressions in default values of typed function + arguments (#60) + +- only return exit code 1 when --check is used (#50) + +- don't remove single trailing commas from square bracket indexing (#59) + +- don't omit whitespace if the previous factor leaf wasn't a math operator (#55) + +- omit extra space in kwarg unpacking if it's the first argument (#46) + +- omit extra space in + [Sphinx auto-attribute comments](http://www.sphinx-doc.org/en/stable/ext/autodoc.html#directive-autoattribute) + (#68) + +## 18.3a3 + +- don't remove single empty lines outside of bracketed expressions (#19) + +- added ability to pipe formatting from stdin to stdin (#25) + +- restored ability to format code with legacy usage of `async` as a name (#20, #42) + +- even better handling of numpy-style array indexing (#33, again) + +## 18.3a2 + +- changed positioning of binary operators to occur at beginning of lines instead of at + the end, following + [a recent change to PEP 8](https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b) + (#21) + +- ignore empty bracket pairs while splitting. This avoids very weirdly looking + formattings (#34, #35) + +- remove a trailing comma if there is a single argument to a call + +- if top level functions were separated by a comment, don't put four empty lines after + the upper function + +- fixed unstable formatting of newlines with imports + +- fixed unintentional folding of post scriptum standalone comments into last statement + if it was a simple statement (#18, #28) + +- fixed missing space in numpy-style array indexing (#33) + +- fixed spurious space after star-based unary expressions (#31) + +## 18.3a1 + +- added `--check` + +- only put trailing commas in function signatures and calls if it's safe to do so. If + the file is Python 3.6+ it's always safe, otherwise only safe if there are no `*args` + or `**kwargs` used in the signature or call. (#8) + +- fixed invalid spacing of dots in relative imports (#6, #13) + +- fixed invalid splitting after comma on unpacked variables in for-loops (#23) + +- fixed spurious space in parenthesized set expressions (#7) + +- fixed spurious space after opening parentheses and in default arguments (#14, #17) + +- fixed spurious space after unary operators when the operand was a complex expression + (#15) + +## 18.3a0 + +- first published version, Happy 🍰 Day 2018! + +- alpha quality + +- date-versioned (see: <https://calver.org/>) diff --git a/contrib/python/black/.dist-info/entry_points.txt b/contrib/python/black/.dist-info/entry_points.txt new file mode 100644 index 00000000000..864da49beba --- /dev/null +++ b/contrib/python/black/.dist-info/entry_points.txt @@ -0,0 +1,6 @@ +[console_scripts] +black = black:patched_main +blackd = blackd:patched_main [d] + +[validate_pyproject.tool_schema] +black = black.schema:get_schema diff --git a/contrib/python/black/.dist-info/top_level.txt b/contrib/python/black/.dist-info/top_level.txt new file mode 100644 index 00000000000..05a77f43969 --- /dev/null +++ b/contrib/python/black/.dist-info/top_level.txt @@ -0,0 +1,4 @@ +_black_version +black +blackd +blib2to3 diff --git a/contrib/python/black/AUTHORS.md b/contrib/python/black/AUTHORS.md new file mode 100644 index 00000000000..fa1ce8f7112 --- /dev/null +++ b/contrib/python/black/AUTHORS.md @@ -0,0 +1,197 @@ +# Authors + +Glued together by [Łukasz Langa](mailto:[email protected]). + +Maintained with: + +- [Carol Willing](mailto:[email protected]) +- [Carl Meyer](mailto:[email protected]) +- [Jelle Zijlstra](mailto:[email protected]) +- [Mika Naylor](mailto:[email protected]) +- [Zsolt Dollenstein](mailto:[email protected]) +- [Cooper Lees](mailto:[email protected]) +- [Richard Si](mailto:[email protected]) +- [Felix Hildén](mailto:[email protected]) +- [Batuhan Taskaya](mailto:[email protected]) +- [Shantanu Jain](mailto:[email protected]) + +Multiple contributions by: + +- [Abdur-Rahmaan Janhangeer](mailto:[email protected]) +- [Adam Johnson](mailto:[email protected]) +- [Adam Williamson](mailto:[email protected]) +- [Alexander Huynh](mailto:[email protected]) +- [Alexandr Artemyev](mailto:[email protected]) +- [Alex Vandiver](mailto:[email protected]) +- [Allan Simon](mailto:[email protected]) +- Anders-Petter Ljungquist +- [Amethyst Reese](mailto:[email protected]) +- [Andrew Thorp](mailto:[email protected]) +- [Andrew Zhou](mailto:[email protected]) +- [Andrey](mailto:[email protected]) +- [Andy Freeland](mailto:[email protected]) +- [Anthony Sottile](mailto:[email protected]) +- [Antonio Ossa Guerra](mailto:[email protected]) +- [Arjaan Buijk](mailto:[email protected]) +- [Arnav Borbornah](mailto:[email protected]) +- [Artem Malyshev](mailto:[email protected]) +- [Asger Hautop Drewsen](mailto:[email protected]) +- [Augie Fackler](mailto:[email protected]) +- [Aviskar KC](mailto:[email protected]) +- Batuhan Taşkaya +- [Benjamin Wohlwend](mailto:[email protected]) +- [Benjamin Woodruff](mailto:[email protected]) +- [Bharat Raghunathan](mailto:[email protected]) +- [Brandt Bucher](mailto:[email protected]) +- [Brett Cannon](mailto:[email protected]) +- [Bryan Bugyi](mailto:[email protected]) +- [Bryan Forbes](mailto:[email protected]) +- [Calum Lind](mailto:[email protected]) +- [Charles](mailto:[email protected]) +- Charles Reid +- [Christian Clauss](mailto:[email protected]) +- [Christian Heimes](mailto:[email protected]) +- [Chuck Wooters](mailto:[email protected]) +- [Chris Rose](mailto:[email protected]) +- Codey Oxley +- [Cong](mailto:[email protected]) +- [Cooper Ry Lees](mailto:[email protected]) +- [Dan Davison](mailto:[email protected]) +- [Daniel Hahler](mailto:[email protected]) +- [Daniel M. Capella](mailto:[email protected]) +- Daniele Esposti +- [David Hotham](mailto:[email protected]) +- [David Lukes](mailto:[email protected]) +- [David Szotten](mailto:[email protected]) +- [Denis Laxalde](mailto:[email protected]) +- [Douglas Thor](mailto:[email protected]) +- dylanjblack +- [Eli Treuherz](mailto:[email protected]) +- [Emil Hessman](mailto:[email protected]) +- [Felix Kohlgrüber](mailto:[email protected]) +- [Florent Thiery](mailto:[email protected]) +- Francisco +- [Giacomo Tagliabue](mailto:[email protected]) +- [Greg Gandenberger](mailto:[email protected]) +- [Gregory P. Smith](mailto:[email protected]) +- Gustavo Camargo +- hauntsaninja +- [Hadi Alqattan](mailto:[email protected]) +- [Hassan Abouelela](mailto:[email protected]) +- [Heaford](mailto:[email protected]) +- [Hugo Barrera](mailto::[email protected]) +- Hugo van Kemenade +- [Hynek Schlawack](mailto:[email protected]) +- [Ionite](mailto:[email protected]) +- [Ivan Katanić](mailto:[email protected]) +- [Jakub Kadlubiec](mailto:[email protected]) +- [Jakub Warczarek](mailto:[email protected]) +- [Jan Hnátek](mailto:[email protected]) +- [Jason Fried](mailto:[email protected]) +- [Jason Friedland](mailto:[email protected]) +- [jgirardet](mailto:[email protected]) +- Jim Brännlund +- [Jimmy Jia](mailto:[email protected]) +- [Joe Antonakakis](mailto:[email protected]) +- [Jon Dufresne](mailto:[email protected]) +- [Jonas Obrist](mailto:[email protected]) +- [Jonty Wareing](mailto:[email protected]) +- [Jose Nazario](mailto:[email protected]) +- [Joseph Larson](mailto:[email protected]) +- [Josh Bode](mailto:[email protected]) +- [Josh Holland](mailto:[email protected]) +- [Joshua Cannon](mailto:[email protected]) +- [José Padilla](mailto:[email protected]) +- [Juan Luis Cano Rodríguez](mailto:[email protected]) +- [kaiix](mailto:[email protected]) +- [Katie McLaughlin](mailto:[email protected]) +- Katrin Leinweber +- [Keith Smiley](mailto:[email protected]) +- [Kenyon Ralph](mailto:[email protected]) +- [Kevin Kirsche](mailto:[email protected]) +- [Kyle Hausmann](mailto:[email protected]) +- [Kyle Sunden](mailto:[email protected]) +- Lawrence Chan +- [Linus Groh](mailto:[email protected]) +- [Loren Carvalho](mailto:[email protected]) +- [Luka Sterbic](mailto:[email protected]) +- [LukasDrude](mailto:[email protected]) +- Mahmoud Hossam +- Mariatta +- [Matt VanEseltine](mailto:[email protected]) +- [Matthew Clapp](mailto:[email protected]) +- [Matthew Walster](mailto:[email protected]) +- Max Smolens +- [Michael Aquilina](mailto:[email protected]) +- [Michael Flaxman](mailto:[email protected]) +- [Michael J. Sullivan](mailto:[email protected]) +- [Michael McClimon](mailto:[email protected]) +- [Miguel Gaiowski](mailto:[email protected]) +- [Mike](mailto:[email protected]) +- [mikehoyio](mailto:[email protected]) +- [Min ho Kim](mailto:[email protected]) +- [Miroslav Shubernetskiy](mailto:[email protected]) +- MomIsBestFriend +- [Nathan Goldbaum](mailto:[email protected]) +- [Nathan Hunt](mailto:[email protected]) +- [Neraste](mailto:[email protected]) +- [Nikolaus Waxweiler](mailto:[email protected]) +- [Ofek Lev](mailto:[email protected]) +- [Osaetin Daniel](mailto:[email protected]) +- [otstrel](mailto:[email protected]) +- [Pablo Galindo](mailto:[email protected]) +- [Paul Ganssle](mailto:[email protected]) +- [Paul Meinhardt](mailto:[email protected]) +- [Peter Bengtsson](mailto:[email protected]) +- [Peter Grayson](mailto:[email protected]) +- [Peter Stensmyr](mailto:[email protected]) +- pmacosta +- [Quentin Pradet](mailto:[email protected]) +- [Ralf Schmitt](mailto:[email protected]) +- [Ramón Valles](mailto:[email protected]) +- [Richard Fearn](mailto:[email protected]) +- [Rishikesh Jha](mailto:[email protected]) +- [Rupert Bedford](mailto:[email protected]) +- Russell Davis +- [Sagi Shadur](mailto:[email protected]) +- [Rémi Verschelde](mailto:[email protected]) +- [Sami Salonen](mailto:[email protected]) +- [Samuel Cormier-Iijima](mailto:[email protected]) +- [Sanket Dasgupta](mailto:[email protected]) +- Sergi +- [Scott Stevenson](mailto:[email protected]) +- Shantanu +- [shaoran](mailto:[email protected]) +- [Shinya Fujino](mailto:[email protected]) +- springstan +- [Stavros Korokithakis](mailto:[email protected]) +- [Stephen Rosen](mailto:[email protected]) +- [Steven M. Vascellaro](mailto:[email protected]) +- [Sunil Kapil](mailto:[email protected]) +- [Sébastien Eustace](mailto:[email protected]) +- [Tal Amuyal](mailto:[email protected]) +- [Terrance](mailto:[email protected]) +- [Thom Lu](mailto:[email protected]) +- [Thomas Grainger](mailto:[email protected]) +- [Tim Gates](mailto:[email protected]) +- [Tim Swast](mailto:[email protected]) +- [Timo](mailto:[email protected]) +- Toby Fleming +- [Tom Christie](mailto:[email protected]) +- [Tony Narlock](mailto:[email protected]) +- [Tsuyoshi Hombashi](mailto:[email protected]) +- [Tushar Chandra](mailto:[email protected]) +- [Tushar Sadhwani](mailto:[email protected]) +- [Tzu-ping Chung](mailto:[email protected]) +- [Utsav Shah](mailto:[email protected]) +- utsav-dbx +- vezeli +- [Ville Skyttä](mailto:[email protected]) +- [Vishwas B Sharma](mailto:[email protected]) +- [Vlad Emelianov](mailto:[email protected]) +- [williamfzc](mailto:[email protected]) +- [wouter bolsterlee](mailto:[email protected]) +- Yazdan +- [Yngve Høiseth](mailto:[email protected]) +- [Yurii Karabas](mailto:[email protected]) +- [Zac Hatfield-Dodds](mailto:[email protected]) diff --git a/contrib/python/black/LICENSE b/contrib/python/black/LICENSE new file mode 100644 index 00000000000..7a9b891f713 --- /dev/null +++ b/contrib/python/black/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Łukasz Langa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/python/black/README.md b/contrib/python/black/README.md new file mode 100644 index 00000000000..cb3cf71f3dc --- /dev/null +++ b/contrib/python/black/README.md @@ -0,0 +1,230 @@ +[](https://black.readthedocs.io/en/stable/) + +<h2 align="center">The Uncompromising Code Formatter</h2> + +<p align="center"> +<a href="https://github.com/psf/black/actions"><img alt="Actions Status" src="https://github.com/psf/black/workflows/Test/badge.svg"></a> +<a href="https://black.readthedocs.io/en/stable/?badge=stable"><img alt="Documentation Status" src="https://readthedocs.org/projects/black/badge/?version=stable"></a> +<a href="https://coveralls.io/github/psf/black?branch=main"><img alt="Coverage Status" src="https://coveralls.io/repos/github/psf/black/badge.svg?branch=main"></a> +<a href="https://github.com/psf/black/blob/main/LICENSE"><img alt="License: MIT" src="https://black.readthedocs.io/en/stable/_static/license.svg"></a> +<a href="https://pypi.org/project/black/"><img alt="PyPI" src="https://img.shields.io/pypi/v/black"></a> +<a href="https://pepy.tech/project/black"><img alt="Downloads" src="https://static.pepy.tech/badge/black"></a> +<a href="https://anaconda.org/conda-forge/black/"><img alt="conda-forge" src="https://img.shields.io/conda/dn/conda-forge/black.svg?label=conda-forge"></a> +<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a> +</p> + +> “Any color you like.” + +_Black_ is the uncompromising Python code formatter. By using it, you agree to cede +control over minutiae of hand-formatting. In return, _Black_ gives you speed, +determinism, and freedom from `pycodestyle` nagging about formatting. You will save time +and mental energy for more important matters. + +Blackened code looks the same regardless of the project you're reading. Formatting +becomes transparent after a while and you can focus on the content instead. + +_Black_ makes code review faster by producing the smallest diffs possible. + +Try it out now using the [Black Playground](https://black.vercel.app). Watch the +[PyCon 2019 talk](https://youtu.be/esZLCuWs_2Y) to learn more. + +--- + +**[Read the documentation on ReadTheDocs!](https://black.readthedocs.io/en/stable)** + +--- + +## Installation and usage + +### Installation + +_Black_ can be installed by running `pip install black`. It requires Python 3.9+ to run. +If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. + +If you can't wait for the latest _hotness_ and want to install from GitHub, use: + +`pip install git+https://github.com/psf/black` + +### Usage + +To get started right away with sensible defaults: + +```sh +black {source_file_or_directory} +``` + +You can run _Black_ as a package if running it as a script doesn't work: + +```sh +python -m black {source_file_or_directory} +``` + +Further information can be found in our docs: + +- [Usage and Configuration](https://black.readthedocs.io/en/stable/usage_and_configuration/index.html) + +_Black_ is already [successfully used](https://github.com/psf/black#used-by) by many +projects, small and big. _Black_ has a comprehensive test suite, with efficient parallel +tests, and our own auto formatting and parallel Continuous Integration runner. Now that +we have become stable, you should not expect large formatting changes in the future. +Stylistic changes will mostly be responses to bug reports and support for new Python +syntax. For more information please refer to +[The Black Code Style](https://black.readthedocs.io/en/stable/the_black_code_style/index.html). + +Also, as a safety measure which slows down processing, _Black_ will check that the +reformatted code still produces a valid AST that is effectively equivalent to the +original (see the +[Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#ast-before-and-after-formatting) +section for details). If you're feeling confident, use `--fast`. + +## The _Black_ code style + +_Black_ is a PEP 8 compliant opinionated formatter. _Black_ reformats entire files in +place. Style configuration options are deliberately limited and rarely added. It doesn't +take previous formatting into account (see +[Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#pragmatism) +for exceptions). + +Our documentation covers the current _Black_ code style, but planned changes to it are +also documented. They're both worth taking a look at: + +- [The _Black_ Code Style: Current style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) +- [The _Black_ Code Style: Future style](https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html) + +Changes to the _Black_ code style are bound by the Stability Policy: + +- [The _Black_ Code Style: Stability Policy](https://black.readthedocs.io/en/stable/the_black_code_style/index.html#stability-policy) + +Please refer to this document before submitting an issue. What seems like a bug might be +intended behaviour. + +### Pragmatism + +Early versions of _Black_ used to be absolutist in some respects. They took after its +initial author. This was fine at the time as it made the implementation simpler and +there were not many users anyway. Not many edge cases were reported. As a mature tool, +_Black_ does make some exceptions to rules it otherwise holds. + +- [The _Black_ code style: Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#pragmatism) + +Please refer to this document before submitting an issue just like with the document +above. What seems like a bug might be intended behaviour. + +## Configuration + +_Black_ is able to read project-specific default values for its command line options +from a `pyproject.toml` file. This is especially useful for specifying custom +`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your +project. + +You can find more details in our documentation: + +- [The basics: Configuration via a file](https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-via-a-file) + +And if you're looking for more general configuration documentation: + +- [Usage and Configuration](https://black.readthedocs.io/en/stable/usage_and_configuration/index.html) + +**Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is +"No". _Black_ is all about sensible defaults. Applying those defaults will have your +code in compliance with many other _Black_ formatted projects. + +## Used by + +The following notable open-source projects trust _Black_ with enforcing a consistent +code style: pytest, tox, Pyramid, Django, Django Channels, Hypothesis, attrs, +SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtualenv), +pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, +Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. + +The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, +Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. + +Are we missing anyone? Let us know. + +## Testimonials + +**Mike Bayer**, [author of `SQLAlchemy`](https://www.sqlalchemy.org/): + +> I can't think of any single tool in my entire programming career that has given me a +> bigger productivity increase by its introduction. I can now do refactorings in about +> 1% of the keystrokes that it would have taken me previously when we had no way for +> code to format itself. + +**Dusty Phillips**, +[writer](https://smile.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords=dusty+phillips): + +> _Black_ is opinionated so you don't have to be. + +**Hynek Schlawack**, [creator of `attrs`](https://www.attrs.org/), core developer of +Twisted and CPython: + +> An auto-formatter that doesn't suck is all I want for Xmas! + +**Carl Meyer**, [Django](https://www.djangoproject.com/) core developer: + +> At least the name is good. + +**Kenneth Reitz**, creator of [`requests`](https://requests.readthedocs.io/en/latest/) +and [`pipenv`](https://readthedocs.org/projects/pipenv/): + +> This vastly improves the formatting of our code. Thanks a ton! + +## Show your style + +Use the badge in your project's README.md: + +```md +[](https://github.com/psf/black) +``` + +Using the badge in README.rst: + +``` +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black +``` + +Looks like this: +[](https://github.com/psf/black) + +## License + +MIT + +## Contributing + +Welcome! Happy to see you willing to make the project better. You can get started by +reading this: + +- [Contributing: The basics](https://black.readthedocs.io/en/latest/contributing/the_basics.html) + +You can also take a look at the rest of the contributing docs or talk with the +developers: + +- [Contributing documentation](https://black.readthedocs.io/en/latest/contributing/index.html) +- [Chat on Discord](https://discord.gg/RtVdv86PrH) + +## Change log + +The log has become rather long. It moved to its own file. + +See [CHANGES](https://black.readthedocs.io/en/latest/change_log.html). + +## Authors + +The author list is quite long nowadays, so it lives in its own file. + +See [AUTHORS.md](./AUTHORS.md) + +## Code of Conduct + +Everyone participating in the _Black_ project, and in particular in the issue tracker, +pull requests, and social media activity, is expected to treat other people with respect +and more generally to follow the guidelines articulated in the +[Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/). + +At the same time, humor is encouraged. In fact, basic familiarity with Monty Python's +Flying Circus is expected. We are not savages. + +And if you _really_ need to slap somebody, do it with a fish while dancing. diff --git a/contrib/python/black/_black_version.py b/contrib/python/black/_black_version.py new file mode 100644 index 00000000000..bc98431b02b --- /dev/null +++ b/contrib/python/black/_black_version.py @@ -0,0 +1 @@ +version = "25.1.0" diff --git a/contrib/python/black/bin/ya.make b/contrib/python/black/bin/ya.make new file mode 100644 index 00000000000..a3e0ee23fca --- /dev/null +++ b/contrib/python/black/bin/ya.make @@ -0,0 +1,15 @@ +PY3_PROGRAM(black) + +SUBSCRIBER(g:python-contrib) + +VERSION(Service-proxy-version) + +LICENSE(MIT) + +PEERDIR( + contrib/python/black +) + +PY_MAIN(black) + +END() diff --git a/contrib/python/black/black/__init__.py b/contrib/python/black/black/__init__.py new file mode 100644 index 00000000000..93a08a8d88a --- /dev/null +++ b/contrib/python/black/black/__init__.py @@ -0,0 +1,1596 @@ +import io +import json +import platform +import re +import sys +import tokenize +import traceback +from collections.abc import ( + Collection, + Generator, + Iterator, + MutableMapping, + Sequence, + Sized, +) +from contextlib import contextmanager +from dataclasses import replace +from datetime import datetime, timezone +from enum import Enum +from json.decoder import JSONDecodeError +from pathlib import Path +from re import Pattern +from typing import Any, Optional, Union + +import click +from click.core import ParameterSource +from mypy_extensions import mypyc_attr +from pathspec import PathSpec +from pathspec.patterns.gitwildmatch import GitWildMatchPatternError + +from _black_version import version as __version__ +from black.cache import Cache +from black.comments import normalize_fmt_off +from black.const import ( + DEFAULT_EXCLUDES, + DEFAULT_INCLUDES, + DEFAULT_LINE_LENGTH, + STDIN_PLACEHOLDER, +) +from black.files import ( + best_effort_relative_path, + find_project_root, + find_pyproject_toml, + find_user_pyproject_toml, + gen_python_files, + get_gitignore, + parse_pyproject_toml, + path_is_excluded, + resolves_outside_root_or_cannot_stat, + wrap_stream_for_windows, +) +from black.handle_ipynb_magics import ( + PYTHON_CELL_MAGICS, + jupyter_dependencies_are_installed, + mask_cell, + put_trailing_semicolon_back, + remove_trailing_semicolon, + unmask_cell, + validate_cell, +) +from black.linegen import LN, LineGenerator, transform_line +from black.lines import EmptyLineTracker, LinesBlock +from black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature +from black.mode import Mode as Mode # re-exported +from black.mode import Preview, TargetVersion, supports_feature +from black.nodes import STARS, is_number_token, is_simple_decorator_expression, syms +from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out +from black.parsing import ( # noqa F401 + ASTSafetyError, + InvalidInput, + lib2to3_parse, + parse_ast, + stringify_ast, +) +from black.ranges import ( + adjusted_lines, + convert_unchanged_lines, + parse_line_ranges, + sanitized_lines, +) +from black.report import Changed, NothingChanged, Report +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node + +COMPILED = Path(__file__).suffix in (".pyd", ".so") + +# types +FileContent = str +Encoding = str +NewLine = str + + +class WriteBack(Enum): + NO = 0 + YES = 1 + DIFF = 2 + CHECK = 3 + COLOR_DIFF = 4 + + @classmethod + def from_configuration( + cls, *, check: bool, diff: bool, color: bool = False + ) -> "WriteBack": + if check and not diff: + return cls.CHECK + + if diff and color: + return cls.COLOR_DIFF + + return cls.DIFF if diff else cls.YES + + +# Legacy name, left for integrations. +FileMode = Mode + + +def read_pyproject_toml( + ctx: click.Context, param: click.Parameter, value: Optional[str] +) -> Optional[str]: + """Inject Black configuration from "pyproject.toml" into defaults in `ctx`. + + Returns the path to a successfully found and read configuration file, None + otherwise. + """ + if not value: + value = find_pyproject_toml( + ctx.params.get("src", ()), ctx.params.get("stdin_filename", None) + ) + if value is None: + return None + + try: + config = parse_pyproject_toml(value) + except (OSError, ValueError) as e: + raise click.FileError( + filename=value, hint=f"Error reading configuration file: {e}" + ) from None + + if not config: + return None + else: + spellcheck_pyproject_toml_keys(ctx, list(config), value) + # Sanitize the values to be Click friendly. For more information please see: + # https://github.com/psf/black/issues/1458 + # https://github.com/pallets/click/issues/1567 + config = { + k: str(v) if not isinstance(v, (list, dict)) else v + for k, v in config.items() + } + + target_version = config.get("target_version") + if target_version is not None and not isinstance(target_version, list): + raise click.BadOptionUsage( + "target-version", "Config key target-version must be a list" + ) + + exclude = config.get("exclude") + if exclude is not None and not isinstance(exclude, str): + raise click.BadOptionUsage("exclude", "Config key exclude must be a string") + + extend_exclude = config.get("extend_exclude") + if extend_exclude is not None and not isinstance(extend_exclude, str): + raise click.BadOptionUsage( + "extend-exclude", "Config key extend-exclude must be a string" + ) + + line_ranges = config.get("line_ranges") + if line_ranges is not None: + raise click.BadOptionUsage( + "line-ranges", "Cannot use line-ranges in the pyproject.toml file." + ) + + default_map: dict[str, Any] = {} + if ctx.default_map: + default_map.update(ctx.default_map) + default_map.update(config) + + ctx.default_map = default_map + return value + + +def spellcheck_pyproject_toml_keys( + ctx: click.Context, config_keys: list[str], config_file_path: str +) -> None: + invalid_keys: list[str] = [] + available_config_options = {param.name for param in ctx.command.params} + for key in config_keys: + if key not in available_config_options: + invalid_keys.append(key) + if invalid_keys: + keys_str = ", ".join(map(repr, invalid_keys)) + out( + f"Invalid config keys detected: {keys_str} (in {config_file_path})", + fg="red", + ) + + +def target_version_option_callback( + c: click.Context, p: Union[click.Option, click.Parameter], v: tuple[str, ...] +) -> list[TargetVersion]: + """Compute the target versions from a --target-version flag. + + This is its own function because mypy couldn't infer the type correctly + when it was a lambda, causing mypyc trouble. + """ + return [TargetVersion[val.upper()] for val in v] + + +def enable_unstable_feature_callback( + c: click.Context, p: Union[click.Option, click.Parameter], v: tuple[str, ...] +) -> list[Preview]: + """Compute the features from an --enable-unstable-feature flag.""" + return [Preview[val] for val in v] + + +def re_compile_maybe_verbose(regex: str) -> Pattern[str]: + """Compile a regular expression string in `regex`. + + If it contains newlines, use verbose mode. + """ + if "\n" in regex: + regex = "(?x)" + regex + compiled: Pattern[str] = re.compile(regex) + return compiled + + +def validate_regex( + ctx: click.Context, + param: click.Parameter, + value: Optional[str], +) -> Optional[Pattern[str]]: + try: + return re_compile_maybe_verbose(value) if value is not None else None + except re.error as e: + raise click.BadParameter(f"Not a valid regular expression: {e}") from None + + + context_settings={"help_option_names": ["-h", "--help"]}, + # While Click does set this field automatically using the docstring, mypyc + # (annoyingly) strips 'em so we need to set it here too. + help="The uncompromising code formatter.", +) [email protected]("-c", "--code", type=str, help="Format the code passed in as a string.") + "-l", + "--line-length", + type=int, + default=DEFAULT_LINE_LENGTH, + help="How many characters per line to allow.", + show_default=True, +) + "-t", + "--target-version", + type=click.Choice([v.name.lower() for v in TargetVersion]), + callback=target_version_option_callback, + multiple=True, + help=( + "Python versions that should be supported by Black's output. You should" + " include all versions that your code supports. By default, Black will infer" + " target versions from the project metadata in pyproject.toml. If this does" + " not yield conclusive results, Black will use per-file auto-detection." + ), +) + "--pyi", + is_flag=True, + help=( + "Format all input files like typing stubs regardless of file extension. This" + " is useful when piping source on standard input." + ), +) + "--ipynb", + is_flag=True, + help=( + "Format all input files like Jupyter Notebooks regardless of file extension." + " This is useful when piping source on standard input." + ), +) + "--python-cell-magics", + multiple=True, + help=( + "When processing Jupyter Notebooks, add the given magic to the list" + f" of known python-magics ({', '.join(sorted(PYTHON_CELL_MAGICS))})." + " Useful for formatting cells with custom python magics." + ), + default=[], +) + "-x", + "--skip-source-first-line", + is_flag=True, + help="Skip the first line of the source code.", +) + "-S", + "--skip-string-normalization", + is_flag=True, + help="Don't normalize string quotes or prefixes.", +) + "-C", + "--skip-magic-trailing-comma", + is_flag=True, + help="Don't use trailing commas as a reason to split lines.", +) + "--preview", + is_flag=True, + help=( + "Enable potentially disruptive style changes that may be added to Black's main" + " functionality in the next major release." + ), +) + "--unstable", + is_flag=True, + help=( + "Enable potentially disruptive style changes that have known bugs or are not" + " currently expected to make it into the stable style Black's next major" + " release. Implies --preview." + ), +) + "--enable-unstable-feature", + type=click.Choice([v.name for v in Preview]), + callback=enable_unstable_feature_callback, + multiple=True, + help=( + "Enable specific features included in the `--unstable` style. Requires" + " `--preview`. No compatibility guarantees are provided on the behavior" + " or existence of any unstable features." + ), +) + "--check", + is_flag=True, + help=( + "Don't write the files back, just return the status. Return code 0 means" + " nothing would change. Return code 1 means some files would be reformatted." + " Return code 123 means there was an internal error." + ), +) + "--diff", + is_flag=True, + help=( + "Don't write the files back, just output a diff to indicate what changes" + " Black would've made. They are printed to stdout so capturing them is simple." + ), +) + "--color/--no-color", + is_flag=True, + help="Show (or do not show) colored diff. Only applies when --diff is given.", +) + "--line-ranges", + multiple=True, + metavar="START-END", + help=( + "When specified, Black will try its best to only format these lines. This" + " option can be specified multiple times, and a union of the lines will be" + " formatted. Each range must be specified as two integers connected by a `-`:" + " `<START>-<END>`. The `<START>` and `<END>` integer indices are 1-based and" + " inclusive on both ends." + ), + default=(), +) + "--fast/--safe", + is_flag=True, + help=( + "By default, Black performs an AST safety check after formatting your code." + " The --fast flag turns off this check and the --safe flag explicitly enables" + " it. [default: --safe]" + ), +) + "--required-version", + type=str, + help=( + "Require a specific version of Black to be running. This is useful for" + " ensuring that all contributors to your project are using the same" + " version, because different versions of Black may format code a little" + " differently. This option can be set in a configuration file for consistent" + " results across environments." + ), +) + "--exclude", + type=str, + callback=validate_regex, + help=( + "A regular expression that matches files and directories that should be" + " excluded on recursive searches. An empty value means no paths are excluded." + " Use forward slashes for directories on all platforms (Windows, too)." + " By default, Black also ignores all paths listed in .gitignore. Changing this" + f" value will override all default exclusions. [default: {DEFAULT_EXCLUDES}]" + ), + show_default=False, +) + "--extend-exclude", + type=str, + callback=validate_regex, + help=( + "Like --exclude, but adds additional files and directories on top of the" + " default values instead of overriding them." + ), +) + "--force-exclude", + type=str, + callback=validate_regex, + help=( + "Like --exclude, but files and directories matching this regex will be excluded" + " even when they are passed explicitly as arguments. This is useful when" + " invoking Black programmatically on changed files, such as in a pre-commit" + " hook or editor plugin." + ), +) + "--stdin-filename", + type=str, + is_eager=True, + help=( + "The name of the file when passing it through stdin. Useful to make sure Black" + " will respect the --force-exclude option on some editors that rely on using" + " stdin." + ), +) + "--include", + type=str, + default=DEFAULT_INCLUDES, + callback=validate_regex, + help=( + "A regular expression that matches files and directories that should be" + " included on recursive searches. An empty value means all files are included" + " regardless of the name. Use forward slashes for directories on all platforms" + " (Windows, too). Overrides all exclusions, including from .gitignore and" + " command line options." + ), + show_default=True, +) + "-W", + "--workers", + type=click.IntRange(min=1), + default=None, + help=( + "When Black formats multiple files, it may use a process pool to speed up" + " formatting. This option controls the number of parallel workers. This can" + " also be specified via the BLACK_NUM_WORKERS environment variable. Defaults" + " to the number of CPUs in the system." + ), +) + "-q", + "--quiet", + is_flag=True, + help=( + "Stop emitting all non-critical output. Error messages will still be emitted" + " (which can silenced by 2>/dev/null)." + ), +) + "-v", + "--verbose", + is_flag=True, + help=( + "Emit messages about files that were not changed or were ignored due to" + " exclusion patterns. If Black is using a configuration file, a message" + " detailing which one it is using will be emitted." + ), +) [email protected]_option( + version=__version__, + message=( + f"%(prog)s, %(version)s (compiled: {'yes' if COMPILED else 'no'})\n" + f"Python ({platform.python_implementation()}) {platform.python_version()}" + ), +) + "src", + nargs=-1, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True + ), + is_eager=True, + metavar="SRC ...", +) + "--config", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + allow_dash=False, + path_type=str, + ), + is_eager=True, + callback=read_pyproject_toml, + help="Read configuration options from a configuration file.", +) [email protected]_context +def main( # noqa: C901 + ctx: click.Context, + code: Optional[str], + line_length: int, + target_version: list[TargetVersion], + check: bool, + diff: bool, + line_ranges: Sequence[str], + color: bool, + fast: bool, + pyi: bool, + ipynb: bool, + python_cell_magics: Sequence[str], + skip_source_first_line: bool, + skip_string_normalization: bool, + skip_magic_trailing_comma: bool, + preview: bool, + unstable: bool, + enable_unstable_feature: list[Preview], + quiet: bool, + verbose: bool, + required_version: Optional[str], + include: Pattern[str], + exclude: Optional[Pattern[str]], + extend_exclude: Optional[Pattern[str]], + force_exclude: Optional[Pattern[str]], + stdin_filename: Optional[str], + workers: Optional[int], + src: tuple[str, ...], + config: Optional[str], +) -> None: + """The uncompromising code formatter.""" + ctx.ensure_object(dict) + + assert sys.version_info >= (3, 9), "Black requires Python 3.9+" + if sys.version_info[:3] == (3, 12, 5): + out( + "Python 3.12.5 has a memory safety issue that can cause Black's " + "AST safety checks to fail. " + "Please upgrade to Python 3.12.6 or downgrade to Python 3.12.4" + ) + ctx.exit(1) + + if src and code is not None: + out( + main.get_usage(ctx) + + "\n\n'SRC' and 'code' cannot be passed simultaneously." + ) + ctx.exit(1) + if not src and code is None: + out(main.get_usage(ctx) + "\n\nOne of 'SRC' or 'code' is required.") + ctx.exit(1) + + # It doesn't do anything if --unstable is also passed, so just allow it. + if enable_unstable_feature and not (preview or unstable): + out( + main.get_usage(ctx) + + "\n\n'--enable-unstable-feature' requires '--preview'." + ) + ctx.exit(1) + + root, method = ( + find_project_root(src, stdin_filename) if code is None else (None, None) + ) + ctx.obj["root"] = root + + if verbose: + if root: + out( + f"Identified `{root}` as project root containing a {method}.", + fg="blue", + ) + + if config: + config_source = ctx.get_parameter_source("config") + user_level_config = str(find_user_pyproject_toml()) + if config == user_level_config: + out( + "Using configuration from user-level config at " + f"'{user_level_config}'.", + fg="blue", + ) + elif config_source in ( + ParameterSource.DEFAULT, + ParameterSource.DEFAULT_MAP, + ): + out("Using configuration from project root.", fg="blue") + else: + out(f"Using configuration in '{config}'.", fg="blue") + if ctx.default_map: + for param, value in ctx.default_map.items(): + out(f"{param}: {value}") + + error_msg = "Oh no! 💥 💔 💥" + if ( + required_version + and required_version != __version__ + and required_version != __version__.split(".")[0] + ): + err( + f"{error_msg} The required version `{required_version}` does not match" + f" the running version `{__version__}`!" + ) + ctx.exit(1) + if ipynb and pyi: + err("Cannot pass both `pyi` and `ipynb` flags!") + ctx.exit(1) + + write_back = WriteBack.from_configuration(check=check, diff=diff, color=color) + if target_version: + versions = set(target_version) + else: + # We'll autodetect later. + versions = set() + mode = Mode( + target_versions=versions, + line_length=line_length, + is_pyi=pyi, + is_ipynb=ipynb, + skip_source_first_line=skip_source_first_line, + string_normalization=not skip_string_normalization, + magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, + unstable=unstable, + python_cell_magics=set(python_cell_magics), + enabled_features=set(enable_unstable_feature), + ) + + lines: list[tuple[int, int]] = [] + if line_ranges: + if ipynb: + err("Cannot use --line-ranges with ipynb files.") + ctx.exit(1) + + try: + lines = parse_line_ranges(line_ranges) + except ValueError as e: + err(str(e)) + ctx.exit(1) + + if code is not None: + # Run in quiet mode by default with -c; the extra output isn't useful. + # You can still pass -v to get verbose output. + quiet = True + + report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose) + + if code is not None: + reformat_code( + content=code, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + lines=lines, + ) + else: + assert root is not None # root is only None if code is not None + try: + sources = get_sources( + root=root, + src=src, + quiet=quiet, + verbose=verbose, + include=include, + exclude=exclude, + extend_exclude=extend_exclude, + force_exclude=force_exclude, + report=report, + stdin_filename=stdin_filename, + ) + except GitWildMatchPatternError: + ctx.exit(1) + + path_empty( + sources, + "No Python files are present to be formatted. Nothing to do 😴", + quiet, + verbose, + ctx, + ) + + if len(sources) == 1: + reformat_one( + src=sources.pop(), + fast=fast, + write_back=write_back, + mode=mode, + report=report, + lines=lines, + ) + else: + from black.concurrency import reformat_many + + if lines: + err("Cannot use --line-ranges to format multiple files.") + ctx.exit(1) + reformat_many( + sources=sources, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + workers=workers, + ) + + if verbose or not quiet: + if code is None and (verbose or report.change_count or report.failure_count): + out() + out(error_msg if report.return_code else "All done! ✨ 🍰 ✨") + if code is None: + click.echo(str(report), err=True) + ctx.exit(report.return_code) + + +def get_sources( + *, + root: Path, + src: tuple[str, ...], + quiet: bool, + verbose: bool, + include: Pattern[str], + exclude: Optional[Pattern[str]], + extend_exclude: Optional[Pattern[str]], + force_exclude: Optional[Pattern[str]], + report: "Report", + stdin_filename: Optional[str], +) -> set[Path]: + """Compute the set of files to be formatted.""" + sources: set[Path] = set() + + assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" + using_default_exclude = exclude is None + exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude + gitignore: Optional[dict[Path, PathSpec]] = None + root_gitignore = get_gitignore(root) + + for s in src: + if s == "-" and stdin_filename: + path = Path(stdin_filename) + if path_is_excluded(stdin_filename, force_exclude): + report.path_ignored( + path, + "--stdin-filename matches the --force-exclude regular expression", + ) + continue + is_stdin = True + else: + path = Path(s) + is_stdin = False + + # Compare the logic here to the logic in `gen_python_files`. + if is_stdin or path.is_file(): + if resolves_outside_root_or_cannot_stat(path, root, report): + if verbose: + out(f'Skipping invalid source: "{path}"', fg="red") + continue + + root_relative_path = best_effort_relative_path(path, root).as_posix() + root_relative_path = "/" + root_relative_path + + # Hard-exclude any files that matches the `--force-exclude` regex. + if path_is_excluded(root_relative_path, force_exclude): + report.path_ignored( + path, "matches the --force-exclude regular expression" + ) + continue + + if is_stdin: + path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") + + if path.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + warn=verbose or not quiet + ): + continue + + if verbose: + out(f'Found input source: "{path}"', fg="blue") + sources.add(path) + elif path.is_dir(): + path = root / (path.resolve().relative_to(root)) + if verbose: + out(f'Found input source directory: "{path}"', fg="blue") + + if using_default_exclude: + gitignore = { + root: root_gitignore, + path: get_gitignore(path), + } + sources.update( + gen_python_files( + path.iterdir(), + root, + include, + exclude, + extend_exclude, + force_exclude, + report, + gitignore, + verbose=verbose, + quiet=quiet, + ) + ) + elif s == "-": + if verbose: + out("Found input source stdin", fg="blue") + sources.add(path) + else: + err(f"invalid path: {s}") + + return sources + + +def path_empty( + src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context +) -> None: + """ + Exit if there is no `src` provided for formatting + """ + if not src: + if verbose or not quiet: + out(msg) + ctx.exit(0) + + +def reformat_code( + content: str, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: Report, + *, + lines: Collection[tuple[int, int]] = (), +) -> None: + """ + Reformat and print out `content` without spawning child processes. + Similar to `reformat_one`, but for string content. + + `fast`, `write_back`, and `mode` options are passed to + :func:`format_file_in_place` or :func:`format_stdin_to_stdout`. + """ + path = Path("<string>") + try: + changed = Changed.NO + if format_stdin_to_stdout( + content=content, fast=fast, write_back=write_back, mode=mode, lines=lines + ): + changed = Changed.YES + report.done(path, changed) + except Exception as exc: + if report.verbose: + traceback.print_exc() + report.failed(path, str(exc)) + + +# diff-shades depends on being to monkeypatch this function to operate. I know it's +# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26 +@mypyc_attr(patchable=True) +def reformat_one( + src: Path, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: "Report", + *, + lines: Collection[tuple[int, int]] = (), +) -> None: + """Reformat a single file under `src` without spawning child processes. + + `fast`, `write_back`, and `mode` options are passed to + :func:`format_file_in_place` or :func:`format_stdin_to_stdout`. + """ + try: + changed = Changed.NO + + if str(src) == "-": + is_stdin = True + elif str(src).startswith(STDIN_PLACEHOLDER): + is_stdin = True + # Use the original name again in case we want to print something + # to the user + src = Path(str(src)[len(STDIN_PLACEHOLDER) :]) + else: + is_stdin = False + + if is_stdin: + if src.suffix == ".pyi": + mode = replace(mode, is_pyi=True) + elif src.suffix == ".ipynb": + mode = replace(mode, is_ipynb=True) + if format_stdin_to_stdout( + fast=fast, write_back=write_back, mode=mode, lines=lines + ): + changed = Changed.YES + else: + cache = Cache.read(mode) + if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): + if not cache.is_changed(src): + changed = Changed.CACHED + if changed is not Changed.CACHED and format_file_in_place( + src, fast=fast, write_back=write_back, mode=mode, lines=lines + ): + changed = Changed.YES + if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( + write_back is WriteBack.CHECK and changed is Changed.NO + ): + cache.write([src]) + report.done(src, changed) + except Exception as exc: + if report.verbose: + traceback.print_exc() + report.failed(src, str(exc)) + + +def format_file_in_place( + src: Path, + fast: bool, + mode: Mode, + write_back: WriteBack = WriteBack.NO, + lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy + *, + lines: Collection[tuple[int, int]] = (), +) -> bool: + """Format file under `src` path. Return True if changed. + + If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted + code to the file. + `mode` and `fast` options are passed to :func:`format_file_contents`. + """ + if src.suffix == ".pyi": + mode = replace(mode, is_pyi=True) + elif src.suffix == ".ipynb": + mode = replace(mode, is_ipynb=True) + + then = datetime.fromtimestamp(src.stat().st_mtime, timezone.utc) + header = b"" + with open(src, "rb") as buf: + if mode.skip_source_first_line: + header = buf.readline() + src_contents, encoding, newline = decode_bytes(buf.read()) + try: + dst_contents = format_file_contents( + src_contents, fast=fast, mode=mode, lines=lines + ) + except NothingChanged: + return False + except JSONDecodeError: + raise ValueError( + f"File '{src}' cannot be parsed as valid Jupyter notebook." + ) from None + src_contents = header.decode(encoding) + src_contents + dst_contents = header.decode(encoding) + dst_contents + + if write_back == WriteBack.YES: + with open(src, "w", encoding=encoding, newline=newline) as f: + f.write(dst_contents) + elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): + now = datetime.now(timezone.utc) + src_name = f"{src}\t{then}" + dst_name = f"{src}\t{now}" + if mode.is_ipynb: + diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) + else: + diff_contents = diff(src_contents, dst_contents, src_name, dst_name) + + if write_back == WriteBack.COLOR_DIFF: + diff_contents = color_diff(diff_contents) + + with lock or nullcontext(): + f = io.TextIOWrapper( + sys.stdout.buffer, + encoding=encoding, + newline=newline, + write_through=True, + ) + f = wrap_stream_for_windows(f) + f.write(diff_contents) + f.detach() + + return True + + +def format_stdin_to_stdout( + fast: bool, + *, + content: Optional[str] = None, + write_back: WriteBack = WriteBack.NO, + mode: Mode, + lines: Collection[tuple[int, int]] = (), +) -> bool: + """Format file on stdin. Return True if changed. + + If content is None, it's read from sys.stdin. + + If `write_back` is YES, write reformatted code back to stdout. If it is DIFF, + write a diff to stdout. The `mode` argument is passed to + :func:`format_file_contents`. + """ + then = datetime.now(timezone.utc) + + if content is None: + src, encoding, newline = decode_bytes(sys.stdin.buffer.read()) + else: + src, encoding, newline = content, "utf-8", "" + + dst = src + try: + dst = format_file_contents(src, fast=fast, mode=mode, lines=lines) + return True + + except NothingChanged: + return False + + finally: + f = io.TextIOWrapper( + sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True + ) + if write_back == WriteBack.YES: + # Make sure there's a newline after the content + if dst and dst[-1] != "\n": + dst += "\n" + f.write(dst) + elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): + now = datetime.now(timezone.utc) + src_name = f"STDIN\t{then}" + dst_name = f"STDOUT\t{now}" + d = diff(src, dst, src_name, dst_name) + if write_back == WriteBack.COLOR_DIFF: + d = color_diff(d) + f = wrap_stream_for_windows(f) + f.write(d) + f.detach() + + +def check_stability_and_equivalence( + src_contents: str, + dst_contents: str, + *, + mode: Mode, + lines: Collection[tuple[int, int]] = (), +) -> None: + """Perform stability and equivalence checks. + + Raise AssertionError if source and destination contents are not + equivalent, or if a second pass of the formatter would format the + content differently. + """ + assert_equivalent(src_contents, dst_contents) + assert_stable(src_contents, dst_contents, mode=mode, lines=lines) + + +def format_file_contents( + src_contents: str, + *, + fast: bool, + mode: Mode, + lines: Collection[tuple[int, int]] = (), +) -> FileContent: + """Reformat contents of a file and return new contents. + + If `fast` is False, additionally confirm that the reformatted code is + valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it. + `mode` is passed to :func:`format_str`. + """ + if mode.is_ipynb: + dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) + else: + dst_contents = format_str(src_contents, mode=mode, lines=lines) + if src_contents == dst_contents: + raise NothingChanged + + if not fast and not mode.is_ipynb: + # Jupyter notebooks will already have been checked above. + check_stability_and_equivalence( + src_contents, dst_contents, mode=mode, lines=lines + ) + return dst_contents + + +def format_cell(src: str, *, fast: bool, mode: Mode) -> str: + """Format code in given cell of Jupyter notebook. + + General idea is: + + - if cell has trailing semicolon, remove it; + - if cell has IPython magics, mask them; + - format cell; + - reinstate IPython magics; + - reinstate trailing semicolon (if originally present); + - strip trailing newlines. + + Cells with syntax errors will not be processed, as they + could potentially be automagics or multi-line magics, which + are currently not supported. + """ + validate_cell(src, mode) + src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( + src + ) + try: + masked_src, replacements = mask_cell(src_without_trailing_semicolon) + except SyntaxError: + raise NothingChanged from None + masked_dst = format_str(masked_src, mode=mode) + if not fast: + check_stability_and_equivalence(masked_src, masked_dst, mode=mode) + dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements) + dst = put_trailing_semicolon_back( + dst_without_trailing_semicolon, has_trailing_semicolon + ) + dst = dst.rstrip("\n") + if dst == src: + raise NothingChanged from None + return dst + + +def validate_metadata(nb: MutableMapping[str, Any]) -> None: + """If notebook is marked as non-Python, don't format it. + + All notebook metadata fields are optional, see + https://nbformat.readthedocs.io/en/latest/format_description.html. So + if a notebook has empty metadata, we will try to parse it anyway. + """ + language = nb.get("metadata", {}).get("language_info", {}).get("name", None) + if language is not None and language != "python": + raise NothingChanged from None + + +def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: + """Format Jupyter notebook. + + Operate cell-by-cell, only on code cells, only for Python notebooks. + If the ``.ipynb`` originally had a trailing newline, it'll be preserved. + """ + if not src_contents: + raise NothingChanged + + trailing_newline = src_contents[-1] == "\n" + modified = False + nb = json.loads(src_contents) + validate_metadata(nb) + for cell in nb["cells"]: + if cell.get("cell_type", None) == "code": + try: + src = "".join(cell["source"]) + dst = format_cell(src, fast=fast, mode=mode) + except NothingChanged: + pass + else: + cell["source"] = dst.splitlines(keepends=True) + modified = True + if modified: + dst_contents = json.dumps(nb, indent=1, ensure_ascii=False) + if trailing_newline: + dst_contents = dst_contents + "\n" + return dst_contents + else: + raise NothingChanged + + +def format_str( + src_contents: str, *, mode: Mode, lines: Collection[tuple[int, int]] = () +) -> str: + """Reformat a string and return new contents. + + `mode` determines formatting options, such as how many characters per line are + allowed. Example: + + >>> import black + >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode())) + def f(arg: str = "") -> None: + ... + + A more complex example: + + >>> print( + ... black.format_str( + ... "def f(arg:str='')->None: hey", + ... mode=black.Mode( + ... target_versions={black.TargetVersion.PY36}, + ... line_length=10, + ... string_normalization=False, + ... is_pyi=False, + ... ), + ... ), + ... ) + def f( + arg: str = '', + ) -> None: + hey + + """ + if lines: + lines = sanitized_lines(lines, src_contents) + if not lines: + return src_contents # Nothing to format + dst_contents = _format_str_once(src_contents, mode=mode, lines=lines) + # Forced second pass to work around optional trailing commas (becoming + # forced trailing commas on pass 2) interacting differently with optional + # parentheses. Admittedly ugly. + if src_contents != dst_contents: + if lines: + lines = adjusted_lines(lines, src_contents, dst_contents) + return _format_str_once(dst_contents, mode=mode, lines=lines) + return dst_contents + + +def _format_str_once( + src_contents: str, *, mode: Mode, lines: Collection[tuple[int, int]] = () +) -> str: + src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) + dst_blocks: list[LinesBlock] = [] + if mode.target_versions: + versions = mode.target_versions + else: + future_imports = get_future_imports(src_node) + versions = detect_target_versions(src_node, future_imports=future_imports) + + context_manager_features = { + feature + for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + if supports_feature(versions, feature) + } + normalize_fmt_off(src_node, mode, lines) + if lines: + # This should be called after normalize_fmt_off. + convert_unchanged_lines(src_node, lines) + + line_generator = LineGenerator(mode=mode, features=context_manager_features) + elt = EmptyLineTracker(mode=mode) + split_line_features = { + feature + for feature in { + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + } + if supports_feature(versions, feature) + } + block: Optional[LinesBlock] = None + for current_line in line_generator.visit(src_node): + block = elt.maybe_empty_lines(current_line) + dst_blocks.append(block) + for line in transform_line( + current_line, mode=mode, features=split_line_features + ): + block.content_lines.append(str(line)) + if dst_blocks: + dst_blocks[-1].after = 0 + dst_contents = [] + for block in dst_blocks: + dst_contents.extend(block.all_lines()) + if not dst_contents: + # Use decode_bytes to retrieve the correct source newline (CRLF or LF), + # and check if normalized_content has more than one line + normalized_content, _, newline = decode_bytes(src_contents.encode("utf-8")) + if "\n" in normalized_content: + return newline + return "" + return "".join(dst_contents) + + +def decode_bytes(src: bytes) -> tuple[FileContent, Encoding, NewLine]: + """Return a tuple of (decoded_contents, encoding, newline). + + `newline` is either CRLF or LF but `decoded_contents` is decoded with + universal newlines (i.e. only contains LF). + """ + srcbuf = io.BytesIO(src) + encoding, lines = tokenize.detect_encoding(srcbuf.readline) + if not lines: + return "", encoding, "\n" + + newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n" + srcbuf.seek(0) + with io.TextIOWrapper(srcbuf, encoding) as tiow: + return tiow.read(), encoding, newline + + +def get_features_used( # noqa: C901 + node: Node, *, future_imports: Optional[set[str]] = None +) -> set[Feature]: + """Return a set of (relatively) new Python features used in this file. + + Currently looking for: + - f-strings; + - self-documenting expressions in f-strings (f"{x=}"); + - underscores in numeric literals; + - trailing commas after * or ** in function signatures and calls; + - positional only arguments in function signatures and lambdas; + - assignment expression; + - relaxed decorator syntax; + - usage of __future__ flags (annotations); + - print / exec statements; + - parenthesized context managers; + - match statements; + - except* clause; + - variadic generics; + """ + features: set[Feature] = set() + if future_imports: + features |= { + FUTURE_FLAG_TO_FEATURE[future_import] + for future_import in future_imports + if future_import in FUTURE_FLAG_TO_FEATURE + } + + for n in node.pre_order(): + if n.type == token.FSTRING_START: + features.add(Feature.F_STRINGS) + elif ( + n.type == token.RBRACE + and n.parent is not None + and any(child.type == token.EQUAL for child in n.parent.children) + ): + features.add(Feature.DEBUG_F_STRINGS) + + elif is_number_token(n): + if "_" in n.value: + features.add(Feature.NUMERIC_UNDERSCORES) + + elif n.type == token.SLASH: + if n.parent and n.parent.type in { + syms.typedargslist, + syms.arglist, + syms.varargslist, + }: + features.add(Feature.POS_ONLY_ARGUMENTS) + + elif n.type == token.COLONEQUAL: + features.add(Feature.ASSIGNMENT_EXPRESSIONS) + + elif n.type == syms.decorator: + if len(n.children) > 1 and not is_simple_decorator_expression( + n.children[1] + ): + features.add(Feature.RELAXED_DECORATORS) + + elif ( + n.type in {syms.typedargslist, syms.arglist} + and n.children + and n.children[-1].type == token.COMMA + ): + if n.type == syms.typedargslist: + feature = Feature.TRAILING_COMMA_IN_DEF + else: + feature = Feature.TRAILING_COMMA_IN_CALL + + for ch in n.children: + if ch.type in STARS: + features.add(feature) + + if ch.type == syms.argument: + for argch in ch.children: + if argch.type in STARS: + features.add(feature) + + elif ( + n.type in {syms.return_stmt, syms.yield_expr} + and len(n.children) >= 2 + and n.children[1].type == syms.testlist_star_expr + and any(child.type == syms.star_expr for child in n.children[1].children) + ): + features.add(Feature.UNPACKING_ON_FLOW) + + elif ( + n.type == syms.annassign + and len(n.children) >= 4 + and n.children[3].type == syms.testlist_star_expr + ): + features.add(Feature.ANN_ASSIGN_EXTENDED_RHS) + + elif ( + n.type == syms.with_stmt + and len(n.children) > 2 + and n.children[1].type == syms.atom + ): + atom_children = n.children[1].children + if ( + len(atom_children) == 3 + and atom_children[0].type == token.LPAR + and _contains_asexpr(atom_children[1]) + and atom_children[2].type == token.RPAR + ): + features.add(Feature.PARENTHESIZED_CONTEXT_MANAGERS) + + elif n.type == syms.match_stmt: + features.add(Feature.PATTERN_MATCHING) + + elif ( + n.type == syms.except_clause + and len(n.children) >= 2 + and n.children[1].type == token.STAR + ): + features.add(Feature.EXCEPT_STAR) + + elif n.type in {syms.subscriptlist, syms.trailer} and any( + child.type == syms.star_expr for child in n.children + ): + features.add(Feature.VARIADIC_GENERICS) + + elif ( + n.type == syms.tname_star + and len(n.children) == 3 + and n.children[2].type == syms.star_expr + ): + features.add(Feature.VARIADIC_GENERICS) + + elif n.type in (syms.type_stmt, syms.typeparams): + features.add(Feature.TYPE_PARAMS) + + elif ( + n.type in (syms.typevartuple, syms.paramspec, syms.typevar) + and n.children[-2].type == token.EQUAL + ): + features.add(Feature.TYPE_PARAM_DEFAULTS) + + return features + + +def _contains_asexpr(node: Union[Node, Leaf]) -> bool: + """Return True if `node` contains an as-pattern.""" + if node.type == syms.asexpr_test: + return True + elif node.type == syms.atom: + if ( + len(node.children) == 3 + and node.children[0].type == token.LPAR + and node.children[2].type == token.RPAR + ): + return _contains_asexpr(node.children[1]) + elif node.type == syms.testlist_gexp: + return any(_contains_asexpr(child) for child in node.children) + return False + + +def detect_target_versions( + node: Node, *, future_imports: Optional[set[str]] = None +) -> set[TargetVersion]: + """Detect the version to target based on the nodes used.""" + features = get_features_used(node, future_imports=future_imports) + return { + version for version in TargetVersion if features <= VERSION_TO_FEATURES[version] + } + + +def get_future_imports(node: Node) -> set[str]: + """Return a set of __future__ imports in the file.""" + imports: set[str] = set() + + def get_imports_from_children(children: list[LN]) -> Generator[str, None, None]: + for child in children: + if isinstance(child, Leaf): + if child.type == token.NAME: + yield child.value + + elif child.type == syms.import_as_name: + orig_name = child.children[0] + assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports" + assert orig_name.type == token.NAME, "Invalid syntax parsing imports" + yield orig_name.value + + elif child.type == syms.import_as_names: + yield from get_imports_from_children(child.children) + + else: + raise AssertionError("Invalid syntax parsing imports") + + for child in node.children: + if child.type != syms.simple_stmt: + break + + first_child = child.children[0] + if isinstance(first_child, Leaf): + # Continue looking if we see a docstring; otherwise stop. + if ( + len(child.children) == 2 + and first_child.type == token.STRING + and child.children[1].type == token.NEWLINE + ): + continue + + break + + elif first_child.type == syms.import_from: + module_name = first_child.children[1] + if not isinstance(module_name, Leaf) or module_name.value != "__future__": + break + + imports |= set(get_imports_from_children(first_child.children[3:])) + else: + break + + return imports + + +def _black_info() -> str: + return ( + f"Black {__version__} on " + f"Python ({platform.python_implementation()}) {platform.python_version()}" + ) + + +def assert_equivalent(src: str, dst: str) -> None: + """Raise AssertionError if `src` and `dst` aren't equivalent.""" + try: + src_ast = parse_ast(src) + except Exception as exc: + raise ASTSafetyError( + "cannot use --safe with this file; failed to parse source file AST: " + f"{exc}\n" + "This could be caused by running Black with an older Python version " + "that does not support new syntax used in your source file." + ) from exc + + try: + dst_ast = parse_ast(dst) + except Exception as exc: + log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst) + raise ASTSafetyError( + f"INTERNAL ERROR: {_black_info()} produced invalid code: {exc}. " + "Please report a bug on https://github.com/psf/black/issues. " + f"This invalid output might be helpful: {log}" + ) from None + + src_ast_str = "\n".join(stringify_ast(src_ast)) + dst_ast_str = "\n".join(stringify_ast(dst_ast)) + if src_ast_str != dst_ast_str: + log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst")) + raise ASTSafetyError( + f"INTERNAL ERROR: {_black_info()} produced code that is not equivalent to" + " the source. Please report a bug on https://github.com/psf/black/issues." + f" This diff might be helpful: {log}" + ) from None + + +def assert_stable( + src: str, dst: str, mode: Mode, *, lines: Collection[tuple[int, int]] = () +) -> None: + """Raise AssertionError if `dst` reformats differently the second time.""" + if lines: + # Formatting specified lines requires `adjusted_lines` to map original lines + # to the formatted lines before re-formatting the previously formatted result. + # Due to less-ideal diff algorithm, some edge cases produce incorrect new line + # ranges. Hence for now, we skip the stable check. + # See https://github.com/psf/black/issues/4033 for context. + return + # We shouldn't call format_str() here, because that formats the string + # twice and may hide a bug where we bounce back and forth between two + # versions. + newdst = _format_str_once(dst, mode=mode, lines=lines) + if dst != newdst: + log = dump_to_file( + str(mode), + diff(src, dst, "source", "first pass"), + diff(dst, newdst, "first pass", "second pass"), + ) + raise AssertionError( + f"INTERNAL ERROR: {_black_info()} produced different code on the second" + " pass of the formatter. Please report a bug on" + f" https://github.com/psf/black/issues. This diff might be helpful: {log}" + ) from None + + +@contextmanager +def nullcontext() -> Iterator[None]: + """Return an empty context manager. + + To be used like `nullcontext` in Python 3.7. + """ + yield + + +def patched_main() -> None: + # PyInstaller patches multiprocessing to need freeze_support() even in non-Windows + # environments so just assume we always need to call it if frozen. + if getattr(sys, "frozen", False): + from multiprocessing import freeze_support + + freeze_support() + + main() + + +if __name__ == "__main__": + patched_main() diff --git a/contrib/python/black/black/__main__.py b/contrib/python/black/black/__main__.py new file mode 100644 index 00000000000..19b810b530a --- /dev/null +++ b/contrib/python/black/black/__main__.py @@ -0,0 +1,3 @@ +from black import patched_main + +patched_main() diff --git a/contrib/python/black/black/_width_table.py b/contrib/python/black/black/_width_table.py new file mode 100644 index 00000000000..5f6ff9febc3 --- /dev/null +++ b/contrib/python/black/black/_width_table.py @@ -0,0 +1,478 @@ +# Generated by make_width_table.py +# wcwidth 0.2.6 +# Unicode 15.0.0 +from typing import Final + +WIDTH_TABLE: Final[list[tuple[int, int, int]]] = [ + (0, 0, 0), + (1, 31, -1), + (127, 159, -1), + (768, 879, 0), + (1155, 1161, 0), + (1425, 1469, 0), + (1471, 1471, 0), + (1473, 1474, 0), + (1476, 1477, 0), + (1479, 1479, 0), + (1552, 1562, 0), + (1611, 1631, 0), + (1648, 1648, 0), + (1750, 1756, 0), + (1759, 1764, 0), + (1767, 1768, 0), + (1770, 1773, 0), + (1809, 1809, 0), + (1840, 1866, 0), + (1958, 1968, 0), + (2027, 2035, 0), + (2045, 2045, 0), + (2070, 2073, 0), + (2075, 2083, 0), + (2085, 2087, 0), + (2089, 2093, 0), + (2137, 2139, 0), + (2200, 2207, 0), + (2250, 2273, 0), + (2275, 2306, 0), + (2362, 2362, 0), + (2364, 2364, 0), + (2369, 2376, 0), + (2381, 2381, 0), + (2385, 2391, 0), + (2402, 2403, 0), + (2433, 2433, 0), + (2492, 2492, 0), + (2497, 2500, 0), + (2509, 2509, 0), + (2530, 2531, 0), + (2558, 2558, 0), + (2561, 2562, 0), + (2620, 2620, 0), + (2625, 2626, 0), + (2631, 2632, 0), + (2635, 2637, 0), + (2641, 2641, 0), + (2672, 2673, 0), + (2677, 2677, 0), + (2689, 2690, 0), + (2748, 2748, 0), + (2753, 2757, 0), + (2759, 2760, 0), + (2765, 2765, 0), + (2786, 2787, 0), + (2810, 2815, 0), + (2817, 2817, 0), + (2876, 2876, 0), + (2879, 2879, 0), + (2881, 2884, 0), + (2893, 2893, 0), + (2901, 2902, 0), + (2914, 2915, 0), + (2946, 2946, 0), + (3008, 3008, 0), + (3021, 3021, 0), + (3072, 3072, 0), + (3076, 3076, 0), + (3132, 3132, 0), + (3134, 3136, 0), + (3142, 3144, 0), + (3146, 3149, 0), + (3157, 3158, 0), + (3170, 3171, 0), + (3201, 3201, 0), + (3260, 3260, 0), + (3263, 3263, 0), + (3270, 3270, 0), + (3276, 3277, 0), + (3298, 3299, 0), + (3328, 3329, 0), + (3387, 3388, 0), + (3393, 3396, 0), + (3405, 3405, 0), + (3426, 3427, 0), + (3457, 3457, 0), + (3530, 3530, 0), + (3538, 3540, 0), + (3542, 3542, 0), + (3633, 3633, 0), + (3636, 3642, 0), + (3655, 3662, 0), + (3761, 3761, 0), + (3764, 3772, 0), + (3784, 3790, 0), + (3864, 3865, 0), + (3893, 3893, 0), + (3895, 3895, 0), + (3897, 3897, 0), + (3953, 3966, 0), + (3968, 3972, 0), + (3974, 3975, 0), + (3981, 3991, 0), + (3993, 4028, 0), + (4038, 4038, 0), + (4141, 4144, 0), + (4146, 4151, 0), + (4153, 4154, 0), + (4157, 4158, 0), + (4184, 4185, 0), + (4190, 4192, 0), + (4209, 4212, 0), + (4226, 4226, 0), + (4229, 4230, 0), + (4237, 4237, 0), + (4253, 4253, 0), + (4352, 4447, 2), + (4957, 4959, 0), + (5906, 5908, 0), + (5938, 5939, 0), + (5970, 5971, 0), + (6002, 6003, 0), + (6068, 6069, 0), + (6071, 6077, 0), + (6086, 6086, 0), + (6089, 6099, 0), + (6109, 6109, 0), + (6155, 6157, 0), + (6159, 6159, 0), + (6277, 6278, 0), + (6313, 6313, 0), + (6432, 6434, 0), + (6439, 6440, 0), + (6450, 6450, 0), + (6457, 6459, 0), + (6679, 6680, 0), + (6683, 6683, 0), + (6742, 6742, 0), + (6744, 6750, 0), + (6752, 6752, 0), + (6754, 6754, 0), + (6757, 6764, 0), + (6771, 6780, 0), + (6783, 6783, 0), + (6832, 6862, 0), + (6912, 6915, 0), + (6964, 6964, 0), + (6966, 6970, 0), + (6972, 6972, 0), + (6978, 6978, 0), + (7019, 7027, 0), + (7040, 7041, 0), + (7074, 7077, 0), + (7080, 7081, 0), + (7083, 7085, 0), + (7142, 7142, 0), + (7144, 7145, 0), + (7149, 7149, 0), + (7151, 7153, 0), + (7212, 7219, 0), + (7222, 7223, 0), + (7376, 7378, 0), + (7380, 7392, 0), + (7394, 7400, 0), + (7405, 7405, 0), + (7412, 7412, 0), + (7416, 7417, 0), + (7616, 7679, 0), + (8203, 8207, 0), + (8232, 8238, 0), + (8288, 8291, 0), + (8400, 8432, 0), + (8986, 8987, 2), + (9001, 9002, 2), + (9193, 9196, 2), + (9200, 9200, 2), + (9203, 9203, 2), + (9725, 9726, 2), + (9748, 9749, 2), + (9800, 9811, 2), + (9855, 9855, 2), + (9875, 9875, 2), + (9889, 9889, 2), + (9898, 9899, 2), + (9917, 9918, 2), + (9924, 9925, 2), + (9934, 9934, 2), + (9940, 9940, 2), + (9962, 9962, 2), + (9970, 9971, 2), + (9973, 9973, 2), + (9978, 9978, 2), + (9981, 9981, 2), + (9989, 9989, 2), + (9994, 9995, 2), + (10024, 10024, 2), + (10060, 10060, 2), + (10062, 10062, 2), + (10067, 10069, 2), + (10071, 10071, 2), + (10133, 10135, 2), + (10160, 10160, 2), + (10175, 10175, 2), + (11035, 11036, 2), + (11088, 11088, 2), + (11093, 11093, 2), + (11503, 11505, 0), + (11647, 11647, 0), + (11744, 11775, 0), + (11904, 11929, 2), + (11931, 12019, 2), + (12032, 12245, 2), + (12272, 12283, 2), + (12288, 12329, 2), + (12330, 12333, 0), + (12334, 12350, 2), + (12353, 12438, 2), + (12441, 12442, 0), + (12443, 12543, 2), + (12549, 12591, 2), + (12593, 12686, 2), + (12688, 12771, 2), + (12784, 12830, 2), + (12832, 12871, 2), + (12880, 19903, 2), + (19968, 42124, 2), + (42128, 42182, 2), + (42607, 42610, 0), + (42612, 42621, 0), + (42654, 42655, 0), + (42736, 42737, 0), + (43010, 43010, 0), + (43014, 43014, 0), + (43019, 43019, 0), + (43045, 43046, 0), + (43052, 43052, 0), + (43204, 43205, 0), + (43232, 43249, 0), + (43263, 43263, 0), + (43302, 43309, 0), + (43335, 43345, 0), + (43360, 43388, 2), + (43392, 43394, 0), + (43443, 43443, 0), + (43446, 43449, 0), + (43452, 43453, 0), + (43493, 43493, 0), + (43561, 43566, 0), + (43569, 43570, 0), + (43573, 43574, 0), + (43587, 43587, 0), + (43596, 43596, 0), + (43644, 43644, 0), + (43696, 43696, 0), + (43698, 43700, 0), + (43703, 43704, 0), + (43710, 43711, 0), + (43713, 43713, 0), + (43756, 43757, 0), + (43766, 43766, 0), + (44005, 44005, 0), + (44008, 44008, 0), + (44013, 44013, 0), + (44032, 55203, 2), + (63744, 64255, 2), + (64286, 64286, 0), + (65024, 65039, 0), + (65040, 65049, 2), + (65056, 65071, 0), + (65072, 65106, 2), + (65108, 65126, 2), + (65128, 65131, 2), + (65281, 65376, 2), + (65504, 65510, 2), + (66045, 66045, 0), + (66272, 66272, 0), + (66422, 66426, 0), + (68097, 68099, 0), + (68101, 68102, 0), + (68108, 68111, 0), + (68152, 68154, 0), + (68159, 68159, 0), + (68325, 68326, 0), + (68900, 68903, 0), + (69291, 69292, 0), + (69373, 69375, 0), + (69446, 69456, 0), + (69506, 69509, 0), + (69633, 69633, 0), + (69688, 69702, 0), + (69744, 69744, 0), + (69747, 69748, 0), + (69759, 69761, 0), + (69811, 69814, 0), + (69817, 69818, 0), + (69826, 69826, 0), + (69888, 69890, 0), + (69927, 69931, 0), + (69933, 69940, 0), + (70003, 70003, 0), + (70016, 70017, 0), + (70070, 70078, 0), + (70089, 70092, 0), + (70095, 70095, 0), + (70191, 70193, 0), + (70196, 70196, 0), + (70198, 70199, 0), + (70206, 70206, 0), + (70209, 70209, 0), + (70367, 70367, 0), + (70371, 70378, 0), + (70400, 70401, 0), + (70459, 70460, 0), + (70464, 70464, 0), + (70502, 70508, 0), + (70512, 70516, 0), + (70712, 70719, 0), + (70722, 70724, 0), + (70726, 70726, 0), + (70750, 70750, 0), + (70835, 70840, 0), + (70842, 70842, 0), + (70847, 70848, 0), + (70850, 70851, 0), + (71090, 71093, 0), + (71100, 71101, 0), + (71103, 71104, 0), + (71132, 71133, 0), + (71219, 71226, 0), + (71229, 71229, 0), + (71231, 71232, 0), + (71339, 71339, 0), + (71341, 71341, 0), + (71344, 71349, 0), + (71351, 71351, 0), + (71453, 71455, 0), + (71458, 71461, 0), + (71463, 71467, 0), + (71727, 71735, 0), + (71737, 71738, 0), + (71995, 71996, 0), + (71998, 71998, 0), + (72003, 72003, 0), + (72148, 72151, 0), + (72154, 72155, 0), + (72160, 72160, 0), + (72193, 72202, 0), + (72243, 72248, 0), + (72251, 72254, 0), + (72263, 72263, 0), + (72273, 72278, 0), + (72281, 72283, 0), + (72330, 72342, 0), + (72344, 72345, 0), + (72752, 72758, 0), + (72760, 72765, 0), + (72767, 72767, 0), + (72850, 72871, 0), + (72874, 72880, 0), + (72882, 72883, 0), + (72885, 72886, 0), + (73009, 73014, 0), + (73018, 73018, 0), + (73020, 73021, 0), + (73023, 73029, 0), + (73031, 73031, 0), + (73104, 73105, 0), + (73109, 73109, 0), + (73111, 73111, 0), + (73459, 73460, 0), + (73472, 73473, 0), + (73526, 73530, 0), + (73536, 73536, 0), + (73538, 73538, 0), + (78912, 78912, 0), + (78919, 78933, 0), + (92912, 92916, 0), + (92976, 92982, 0), + (94031, 94031, 0), + (94095, 94098, 0), + (94176, 94179, 2), + (94180, 94180, 0), + (94192, 94193, 2), + (94208, 100343, 2), + (100352, 101589, 2), + (101632, 101640, 2), + (110576, 110579, 2), + (110581, 110587, 2), + (110589, 110590, 2), + (110592, 110882, 2), + (110898, 110898, 2), + (110928, 110930, 2), + (110933, 110933, 2), + (110948, 110951, 2), + (110960, 111355, 2), + (113821, 113822, 0), + (118528, 118573, 0), + (118576, 118598, 0), + (119143, 119145, 0), + (119163, 119170, 0), + (119173, 119179, 0), + (119210, 119213, 0), + (119362, 119364, 0), + (121344, 121398, 0), + (121403, 121452, 0), + (121461, 121461, 0), + (121476, 121476, 0), + (121499, 121503, 0), + (121505, 121519, 0), + (122880, 122886, 0), + (122888, 122904, 0), + (122907, 122913, 0), + (122915, 122916, 0), + (122918, 122922, 0), + (123023, 123023, 0), + (123184, 123190, 0), + (123566, 123566, 0), + (123628, 123631, 0), + (124140, 124143, 0), + (125136, 125142, 0), + (125252, 125258, 0), + (126980, 126980, 2), + (127183, 127183, 2), + (127374, 127374, 2), + (127377, 127386, 2), + (127488, 127490, 2), + (127504, 127547, 2), + (127552, 127560, 2), + (127568, 127569, 2), + (127584, 127589, 2), + (127744, 127776, 2), + (127789, 127797, 2), + (127799, 127868, 2), + (127870, 127891, 2), + (127904, 127946, 2), + (127951, 127955, 2), + (127968, 127984, 2), + (127988, 127988, 2), + (127992, 128062, 2), + (128064, 128064, 2), + (128066, 128252, 2), + (128255, 128317, 2), + (128331, 128334, 2), + (128336, 128359, 2), + (128378, 128378, 2), + (128405, 128406, 2), + (128420, 128420, 2), + (128507, 128591, 2), + (128640, 128709, 2), + (128716, 128716, 2), + (128720, 128722, 2), + (128725, 128727, 2), + (128732, 128735, 2), + (128747, 128748, 2), + (128756, 128764, 2), + (128992, 129003, 2), + (129008, 129008, 2), + (129292, 129338, 2), + (129340, 129349, 2), + (129351, 129535, 2), + (129648, 129660, 2), + (129664, 129672, 2), + (129680, 129725, 2), + (129727, 129733, 2), + (129742, 129755, 2), + (129760, 129768, 2), + (129776, 129784, 2), + (131072, 196605, 2), + (196608, 262141, 2), + (917760, 917999, 0), +] diff --git a/contrib/python/black/black/brackets.py b/contrib/python/black/black/brackets.py new file mode 100644 index 00000000000..c2e8be4348e --- /dev/null +++ b/contrib/python/black/black/brackets.py @@ -0,0 +1,383 @@ +"""Builds on top of nodes.py to track brackets.""" + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import Final, Optional, Union + +from black.nodes import ( + BRACKET, + CLOSING_BRACKETS, + COMPARATORS, + LOGIC_OPERATORS, + MATH_OPERATORS, + OPENING_BRACKETS, + UNPACKING_PARENTS, + VARARGS_PARENTS, + is_vararg, + syms, +) +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node + +# types +LN = Union[Leaf, Node] +Depth = int +LeafID = int +NodeType = int +Priority = int + + +COMPREHENSION_PRIORITY: Final = 20 +COMMA_PRIORITY: Final = 18 +TERNARY_PRIORITY: Final = 16 +LOGIC_PRIORITY: Final = 14 +STRING_PRIORITY: Final = 12 +COMPARATOR_PRIORITY: Final = 10 +MATH_PRIORITIES: Final = { + token.VBAR: 9, + token.CIRCUMFLEX: 8, + token.AMPER: 7, + token.LEFTSHIFT: 6, + token.RIGHTSHIFT: 6, + token.PLUS: 5, + token.MINUS: 5, + token.STAR: 4, + token.SLASH: 4, + token.DOUBLESLASH: 4, + token.PERCENT: 4, + token.AT: 4, + token.TILDE: 3, + token.DOUBLESTAR: 2, +} +DOT_PRIORITY: Final = 1 + + +class BracketMatchError(Exception): + """Raised when an opening bracket is unable to be matched to a closing bracket.""" + + +@dataclass +class BracketTracker: + """Keeps track of brackets on a line.""" + + depth: int = 0 + bracket_match: dict[tuple[Depth, NodeType], Leaf] = field(default_factory=dict) + delimiters: dict[LeafID, Priority] = field(default_factory=dict) + previous: Optional[Leaf] = None + _for_loop_depths: list[int] = field(default_factory=list) + _lambda_argument_depths: list[int] = field(default_factory=list) + invisible: list[Leaf] = field(default_factory=list) + + def mark(self, leaf: Leaf) -> None: + """Mark `leaf` with bracket-related metadata. Keep track of delimiters. + + All leaves receive an int `bracket_depth` field that stores how deep + within brackets a given leaf is. 0 means there are no enclosing brackets + that started on this line. + + If a leaf is itself a closing bracket and there is a matching opening + bracket earlier, it receives an `opening_bracket` field with which it forms a + pair. This is a one-directional link to avoid reference cycles. Closing + bracket without opening happens on lines continued from previous + breaks, e.g. `) -> "ReturnType":` as part of a funcdef where we place + the return type annotation on its own line of the previous closing RPAR. + + If a leaf is a delimiter (a token on which Black can split the line if + needed) and it's on depth 0, its `id()` is stored in the tracker's + `delimiters` field. + """ + if leaf.type == token.COMMENT: + return + + if ( + self.depth == 0 + and leaf.type in CLOSING_BRACKETS + and (self.depth, leaf.type) not in self.bracket_match + ): + return + + self.maybe_decrement_after_for_loop_variable(leaf) + self.maybe_decrement_after_lambda_arguments(leaf) + if leaf.type in CLOSING_BRACKETS: + self.depth -= 1 + try: + opening_bracket = self.bracket_match.pop((self.depth, leaf.type)) + except KeyError as e: + raise BracketMatchError( + "Unable to match a closing bracket to the following opening" + f" bracket: {leaf}" + ) from e + leaf.opening_bracket = opening_bracket + if not leaf.value: + self.invisible.append(leaf) + leaf.bracket_depth = self.depth + if self.depth == 0: + delim = is_split_before_delimiter(leaf, self.previous) + if delim and self.previous is not None: + self.delimiters[id(self.previous)] = delim + else: + delim = is_split_after_delimiter(leaf) + if delim: + self.delimiters[id(leaf)] = delim + if leaf.type in OPENING_BRACKETS: + self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf + self.depth += 1 + if not leaf.value: + self.invisible.append(leaf) + self.previous = leaf + self.maybe_increment_lambda_arguments(leaf) + self.maybe_increment_for_loop_variable(leaf) + + def any_open_for_or_lambda(self) -> bool: + """Return True if there is an open for or lambda expression on the line. + + See maybe_increment_for_loop_variable and maybe_increment_lambda_arguments + for details.""" + return bool(self._for_loop_depths or self._lambda_argument_depths) + + def any_open_brackets(self) -> bool: + """Return True if there is an yet unmatched open bracket on the line.""" + return bool(self.bracket_match) + + def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> Priority: + """Return the highest priority of a delimiter found on the line. + + Values are consistent with what `is_split_*_delimiter()` return. + Raises ValueError on no delimiters. + """ + return max(v for k, v in self.delimiters.items() if k not in exclude) + + def delimiter_count_with_priority(self, priority: Priority = 0) -> int: + """Return the number of delimiters with the given `priority`. + + If no `priority` is passed, defaults to max priority on the line. + """ + if not self.delimiters: + return 0 + + priority = priority or self.max_delimiter_priority() + return sum(1 for p in self.delimiters.values() if p == priority) + + def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: + """In a for loop, or comprehension, the variables are often unpacks. + + To avoid splitting on the comma in this situation, increase the depth of + tokens between `for` and `in`. + """ + if leaf.type == token.NAME and leaf.value == "for": + self.depth += 1 + self._for_loop_depths.append(self.depth) + return True + + return False + + def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool: + """See `maybe_increment_for_loop_variable` above for explanation.""" + if ( + self._for_loop_depths + and self._for_loop_depths[-1] == self.depth + and leaf.type == token.NAME + and leaf.value == "in" + ): + self.depth -= 1 + self._for_loop_depths.pop() + return True + + return False + + def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool: + """In a lambda expression, there might be more than one argument. + + To avoid splitting on the comma in this situation, increase the depth of + tokens between `lambda` and `:`. + """ + if leaf.type == token.NAME and leaf.value == "lambda": + self.depth += 1 + self._lambda_argument_depths.append(self.depth) + return True + + return False + + def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool: + """See `maybe_increment_lambda_arguments` above for explanation.""" + if ( + self._lambda_argument_depths + and self._lambda_argument_depths[-1] == self.depth + and leaf.type == token.COLON + ): + self.depth -= 1 + self._lambda_argument_depths.pop() + return True + + return False + + def get_open_lsqb(self) -> Optional[Leaf]: + """Return the most recent opening square bracket (if any).""" + return self.bracket_match.get((self.depth - 1, token.RSQB)) + + +def is_split_after_delimiter(leaf: Leaf) -> Priority: + """Return the priority of the `leaf` delimiter, given a line break after it. + + The delimiter priorities returned here are from those delimiters that would + cause a line break after themselves. + + Higher numbers are higher priority. + """ + if leaf.type == token.COMMA: + return COMMA_PRIORITY + + return 0 + + +def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: + """Return the priority of the `leaf` delimiter, given a line break before it. + + The delimiter priorities returned here are from those delimiters that would + cause a line break before themselves. + + Higher numbers are higher priority. + """ + if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS): + # * and ** might also be MATH_OPERATORS but in this case they are not. + # Don't treat them as a delimiter. + return 0 + + if ( + leaf.type == token.DOT + and leaf.parent + and leaf.parent.type not in {syms.import_from, syms.dotted_name} + and (previous is None or previous.type in CLOSING_BRACKETS) + ): + return DOT_PRIORITY + + if ( + leaf.type in MATH_OPERATORS + and leaf.parent + and leaf.parent.type not in {syms.factor, syms.star_expr} + ): + return MATH_PRIORITIES[leaf.type] + + if leaf.type in COMPARATORS: + return COMPARATOR_PRIORITY + + if ( + leaf.type == token.STRING + and previous is not None + and previous.type == token.STRING + ): + return STRING_PRIORITY + + if leaf.type not in {token.NAME, token.ASYNC}: + return 0 + + if ( + leaf.value == "for" + and leaf.parent + and leaf.parent.type in {syms.comp_for, syms.old_comp_for} + or leaf.type == token.ASYNC + ): + if ( + not isinstance(leaf.prev_sibling, Leaf) + or leaf.prev_sibling.value != "async" + ): + return COMPREHENSION_PRIORITY + + if ( + leaf.value == "if" + and leaf.parent + and leaf.parent.type in {syms.comp_if, syms.old_comp_if} + ): + return COMPREHENSION_PRIORITY + + if leaf.value in {"if", "else"} and leaf.parent and leaf.parent.type == syms.test: + return TERNARY_PRIORITY + + if leaf.value == "is": + return COMPARATOR_PRIORITY + + if ( + leaf.value == "in" + and leaf.parent + and leaf.parent.type in {syms.comp_op, syms.comparison} + and not ( + previous is not None + and previous.type == token.NAME + and previous.value == "not" + ) + ): + return COMPARATOR_PRIORITY + + if ( + leaf.value == "not" + and leaf.parent + and leaf.parent.type == syms.comp_op + and not ( + previous is not None + and previous.type == token.NAME + and previous.value == "is" + ) + ): + return COMPARATOR_PRIORITY + + if leaf.value in LOGIC_OPERATORS and leaf.parent: + return LOGIC_PRIORITY + + return 0 + + +def max_delimiter_priority_in_atom(node: LN) -> Priority: + """Return maximum delimiter priority inside `node`. + + This is specific to atoms with contents contained in a pair of parentheses. + If `node` isn't an atom or there are no enclosing parentheses, returns 0. + """ + if node.type != syms.atom: + return 0 + + first = node.children[0] + last = node.children[-1] + if not (first.type == token.LPAR and last.type == token.RPAR): + return 0 + + bt = BracketTracker() + for c in node.children[1:-1]: + if isinstance(c, Leaf): + bt.mark(c) + else: + for leaf in c.leaves(): + bt.mark(leaf) + try: + return bt.max_delimiter_priority() + + except ValueError: + return 0 + + +def get_leaves_inside_matching_brackets(leaves: Sequence[Leaf]) -> set[LeafID]: + """Return leaves that are inside matching brackets. + + The input `leaves` can have non-matching brackets at the head or tail parts. + Matching brackets are included. + """ + try: + # Start with the first opening bracket and ignore closing brackets before. + start_index = next( + i for i, l in enumerate(leaves) if l.type in OPENING_BRACKETS + ) + except StopIteration: + return set() + bracket_stack = [] + ids = set() + for i in range(start_index, len(leaves)): + leaf = leaves[i] + if leaf.type in OPENING_BRACKETS: + bracket_stack.append((BRACKET[leaf.type], i)) + if leaf.type in CLOSING_BRACKETS: + if bracket_stack and leaf.type == bracket_stack[-1][0]: + _, start = bracket_stack.pop() + for j in range(start, i + 1): + ids.add(id(leaves[j])) + else: + break + return ids diff --git a/contrib/python/black/black/cache.py b/contrib/python/black/black/cache.py new file mode 100644 index 00000000000..ef9d99a7b90 --- /dev/null +++ b/contrib/python/black/black/cache.py @@ -0,0 +1,150 @@ +"""Caching of formatted files with feature-based invalidation.""" + +import hashlib +import os +import pickle +import sys +import tempfile +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path +from typing import NamedTuple + +from platformdirs import user_cache_dir + +from _black_version import version as __version__ +from black.mode import Mode +from black.output import err + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class FileData(NamedTuple): + st_mtime: float + st_size: int + hash: str + + +def get_cache_dir() -> Path: + """Get the cache directory used by black. + + Users can customize this directory on all systems using `BLACK_CACHE_DIR` + environment variable. By default, the cache directory is the user cache directory + under the black application. + + This result is immediately set to a constant `black.cache.CACHE_DIR` as to avoid + repeated calls. + """ + # NOTE: Function mostly exists as a clean way to test getting the cache directory. + default_cache_dir = user_cache_dir("black") + cache_dir = Path(os.environ.get("BLACK_CACHE_DIR", default_cache_dir)) + cache_dir = cache_dir / __version__ + return cache_dir + + +CACHE_DIR = get_cache_dir() + + +def get_cache_file(mode: Mode) -> Path: + return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle" + + +@dataclass +class Cache: + mode: Mode + cache_file: Path + file_data: dict[str, FileData] = field(default_factory=dict) + + @classmethod + def read(cls, mode: Mode) -> Self: + """Read the cache if it exists and is well-formed. + + If it is not well-formed, the call to write later should + resolve the issue. + """ + cache_file = get_cache_file(mode) + try: + exists = cache_file.exists() + except OSError as e: + # Likely file too long; see #4172 and #4174 + err(f"Unable to read cache file {cache_file} due to {e}") + return cls(mode, cache_file) + if not exists: + return cls(mode, cache_file) + + with cache_file.open("rb") as fobj: + try: + data: dict[str, tuple[float, int, str]] = pickle.load(fobj) + file_data = {k: FileData(*v) for k, v in data.items()} + except (pickle.UnpicklingError, ValueError, IndexError): + return cls(mode, cache_file) + + return cls(mode, cache_file, file_data) + + @staticmethod + def hash_digest(path: Path) -> str: + """Return hash digest for path.""" + + data = path.read_bytes() + return hashlib.sha256(data).hexdigest() + + @staticmethod + def get_file_data(path: Path) -> FileData: + """Return file data for path.""" + + stat = path.stat() + hash = Cache.hash_digest(path) + return FileData(stat.st_mtime, stat.st_size, hash) + + def is_changed(self, source: Path) -> bool: + """Check if source has changed compared to cached version.""" + res_src = source.resolve() + old = self.file_data.get(str(res_src)) + if old is None: + return True + + st = res_src.stat() + if st.st_size != old.st_size: + return True + if st.st_mtime != old.st_mtime: + new_hash = Cache.hash_digest(res_src) + if new_hash != old.hash: + return True + return False + + def filtered_cached(self, sources: Iterable[Path]) -> tuple[set[Path], set[Path]]: + """Split an iterable of paths in `sources` into two sets. + + The first contains paths of files that modified on disk or are not in the + cache. The other contains paths to non-modified files. + """ + changed: set[Path] = set() + done: set[Path] = set() + for src in sources: + if self.is_changed(src): + changed.add(src) + else: + done.add(src) + return changed, done + + def write(self, sources: Iterable[Path]) -> None: + """Update the cache file data and write a new cache file.""" + self.file_data.update( + **{str(src.resolve()): Cache.get_file_data(src) for src in sources} + ) + try: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=str(self.cache_file.parent), delete=False + ) as f: + # We store raw tuples in the cache because it's faster. + data: dict[str, tuple[float, int, str]] = { + k: (*v,) for k, v in self.file_data.items() + } + pickle.dump(data, f, protocol=4) + os.replace(f.name, self.cache_file) + except OSError: + pass diff --git a/contrib/python/black/black/comments.py b/contrib/python/black/black/comments.py new file mode 100644 index 00000000000..f42a51033db --- /dev/null +++ b/contrib/python/black/black/comments.py @@ -0,0 +1,412 @@ +import re +from collections.abc import Collection, Iterator +from dataclasses import dataclass +from functools import lru_cache +from typing import Final, Optional, Union + +from black.mode import Mode +from black.nodes import ( + CLOSING_BRACKETS, + STANDALONE_COMMENT, + WHITESPACE, + container_of, + first_leaf_of, + make_simple_prefix, + preceding_leaf, + syms, +) +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node + +# types +LN = Union[Leaf, Node] + +FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"} +FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"} +FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"} + +COMMENT_EXCEPTIONS = " !:#'" +_COMMENT_PREFIX = "# " +_COMMENT_LIST_SEPARATOR = ";" + + +@dataclass +class ProtoComment: + """Describes a piece of syntax that is a comment. + + It's not a :class:`blib2to3.pytree.Leaf` so that: + + * it can be cached (`Leaf` objects should not be reused more than once as + they store their lineno, column, prefix, and parent information); + * `newlines` and `consumed` fields are kept separate from the `value`. This + simplifies handling of special marker comments like ``# fmt: off/on``. + """ + + type: int # token.COMMENT or STANDALONE_COMMENT + value: str # content of the comment + newlines: int # how many newlines before the comment + consumed: int # how many characters of the original leaf's prefix did we consume + form_feed: bool # is there a form feed before the comment + leading_whitespace: str # leading whitespace before the comment, if any + + +def generate_comments(leaf: LN) -> Iterator[Leaf]: + """Clean the prefix of the `leaf` and generate comments from it, if any. + + Comments in lib2to3 are shoved into the whitespace prefix. This happens + in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation + move because it does away with modifying the grammar to include all the + possible places in which comments can be placed. + + The sad consequence for us though is that comments don't "belong" anywhere. + This is why this function generates simple parentless Leaf objects for + comments. We simply don't know what the correct parent should be. + + No matter though, we can live without this. We really only need to + differentiate between inline and standalone comments. The latter don't + share the line with any code. + + Inline comments are emitted as regular token.COMMENT leaves. Standalone + are emitted with a fake STANDALONE_COMMENT token identifier. + """ + total_consumed = 0 + for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): + total_consumed = pc.consumed + prefix = make_simple_prefix(pc.newlines, pc.form_feed) + yield Leaf(pc.type, pc.value, prefix=prefix) + normalize_trailing_prefix(leaf, total_consumed) + + +@lru_cache(maxsize=4096) +def list_comments(prefix: str, *, is_endmarker: bool) -> list[ProtoComment]: + """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.""" + result: list[ProtoComment] = [] + if not prefix or "#" not in prefix: + return result + + consumed = 0 + nlines = 0 + ignored_lines = 0 + form_feed = False + for index, full_line in enumerate(re.split("\r?\n", prefix)): + consumed += len(full_line) + 1 # adding the length of the split '\n' + match = re.match(r"^(\s*)(\S.*|)$", full_line) + assert match + whitespace, line = match.groups() + if not line: + nlines += 1 + if "\f" in full_line: + form_feed = True + if not line.startswith("#"): + # Escaped newlines outside of a comment are not really newlines at + # all. We treat a single-line comment following an escaped newline + # as a simple trailing comment. + if line.endswith("\\"): + ignored_lines += 1 + continue + + if index == ignored_lines and not is_endmarker: + comment_type = token.COMMENT # simple trailing comment + else: + comment_type = STANDALONE_COMMENT + comment = make_comment(line) + result.append( + ProtoComment( + type=comment_type, + value=comment, + newlines=nlines, + consumed=consumed, + form_feed=form_feed, + leading_whitespace=whitespace, + ) + ) + form_feed = False + nlines = 0 + return result + + +def normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None: + """Normalize the prefix that's left over after generating comments. + + Note: don't use backslashes for formatting or you'll lose your voting rights. + """ + remainder = leaf.prefix[total_consumed:] + if "\\" not in remainder: + nl_count = remainder.count("\n") + form_feed = "\f" in remainder and remainder.endswith("\n") + leaf.prefix = make_simple_prefix(nl_count, form_feed) + return + + leaf.prefix = "" + + +def make_comment(content: str) -> str: + """Return a consistently formatted comment from the given `content` string. + + All comments (except for "##", "#!", "#:", '#'") should have a single + space between the hash sign and the content. + + If `content` didn't start with a hash sign, one is provided. + """ + content = content.rstrip() + if not content: + return "#" + + if content[0] == "#": + content = content[1:] + NON_BREAKING_SPACE = " " + if ( + content + and content[0] == NON_BREAKING_SPACE + and not content.lstrip().startswith("type:") + ): + content = " " + content[1:] # Replace NBSP by a simple space + if content and content[0] not in COMMENT_EXCEPTIONS: + content = " " + content + return "#" + content + + +def normalize_fmt_off( + node: Node, mode: Mode, lines: Collection[tuple[int, int]] +) -> None: + """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" + try_again = True + while try_again: + try_again = convert_one_fmt_off_pair(node, mode, lines) + + +def convert_one_fmt_off_pair( + node: Node, mode: Mode, lines: Collection[tuple[int, int]] +) -> bool: + """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. + + Returns True if a pair was converted. + """ + for leaf in node.leaves(): + previous_consumed = 0 + for comment in list_comments(leaf.prefix, is_endmarker=False): + is_fmt_off = comment.value in FMT_OFF + is_fmt_skip = _contains_fmt_skip_comment(comment.value, mode) + if (not is_fmt_off and not is_fmt_skip) or ( + # Invalid use when `# fmt: off` is applied before a closing bracket. + is_fmt_off + and leaf.type in CLOSING_BRACKETS + ): + previous_consumed = comment.consumed + continue + # We only want standalone comments. If there's no previous leaf or + # the previous leaf is indentation, it's a standalone comment in + # disguise. + if comment.type != STANDALONE_COMMENT: + prev = preceding_leaf(leaf) + if prev: + if is_fmt_off and prev.type not in WHITESPACE: + continue + if is_fmt_skip and prev.type in WHITESPACE: + continue + + ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode)) + if not ignored_nodes: + continue + + first = ignored_nodes[0] # Can be a container node with the `leaf`. + parent = first.parent + prefix = first.prefix + if comment.value in FMT_OFF: + first.prefix = prefix[comment.consumed :] + if is_fmt_skip: + first.prefix = "" + standalone_comment_prefix = prefix + else: + standalone_comment_prefix = ( + prefix[:previous_consumed] + "\n" * comment.newlines + ) + hidden_value = "".join(str(n) for n in ignored_nodes) + comment_lineno = leaf.lineno - comment.newlines + if comment.value in FMT_OFF: + fmt_off_prefix = "" + if len(lines) > 0 and not any( + line[0] <= comment_lineno <= line[1] for line in lines + ): + # keeping indentation of comment by preserving original whitespaces. + fmt_off_prefix = prefix.split(comment.value)[0] + if "\n" in fmt_off_prefix: + fmt_off_prefix = fmt_off_prefix.split("\n")[-1] + standalone_comment_prefix += fmt_off_prefix + hidden_value = comment.value + "\n" + hidden_value + if is_fmt_skip: + hidden_value += comment.leading_whitespace + comment.value + if hidden_value.endswith("\n"): + # That happens when one of the `ignored_nodes` ended with a NEWLINE + # leaf (possibly followed by a DEDENT). + hidden_value = hidden_value[:-1] + first_idx: Optional[int] = None + for ignored in ignored_nodes: + index = ignored.remove() + if first_idx is None: + first_idx = index + assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)" + assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)" + parent.insert_child( + first_idx, + Leaf( + STANDALONE_COMMENT, + hidden_value, + prefix=standalone_comment_prefix, + fmt_pass_converted_first_leaf=first_leaf_of(first), + ), + ) + return True + + return False + + +def generate_ignored_nodes( + leaf: Leaf, comment: ProtoComment, mode: Mode +) -> Iterator[LN]: + """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. + + If comment is skip, returns leaf only. + Stops at the end of the block. + """ + if _contains_fmt_skip_comment(comment.value, mode): + yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment) + return + container: Optional[LN] = container_of(leaf) + while container is not None and container.type != token.ENDMARKER: + if is_fmt_on(container): + return + + # fix for fmt: on in children + if children_contains_fmt_on(container): + for index, child in enumerate(container.children): + if isinstance(child, Leaf) and is_fmt_on(child): + if child.type in CLOSING_BRACKETS: + # This means `# fmt: on` is placed at a different bracket level + # than `# fmt: off`. This is an invalid use, but as a courtesy, + # we include this closing bracket in the ignored nodes. + # The alternative is to fail the formatting. + yield child + return + if ( + child.type == token.INDENT + and index < len(container.children) - 1 + and children_contains_fmt_on(container.children[index + 1]) + ): + # This means `# fmt: on` is placed right after an indentation + # level, and we shouldn't swallow the previous INDENT token. + return + if children_contains_fmt_on(child): + return + yield child + else: + if container.type == token.DEDENT and container.next_sibling is None: + # This can happen when there is no matching `# fmt: on` comment at the + # same level as `# fmt: on`. We need to keep this DEDENT. + return + yield container + container = container.next_sibling + + +def _generate_ignored_nodes_from_fmt_skip( + leaf: Leaf, comment: ProtoComment +) -> Iterator[LN]: + """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.""" + prev_sibling = leaf.prev_sibling + parent = leaf.parent + # Need to properly format the leaf prefix to compare it to comment.value, + # which is also formatted + comments = list_comments(leaf.prefix, is_endmarker=False) + if not comments or comment.value != comments[0].value: + return + if prev_sibling is not None: + leaf.prefix = "" + siblings = [prev_sibling] + while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None: + prev_sibling = prev_sibling.prev_sibling + siblings.insert(0, prev_sibling) + yield from siblings + elif ( + parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE + ): + # The `# fmt: skip` is on the colon line of the if/while/def/class/... + # statements. The ignored nodes should be previous siblings of the + # parent suite node. + leaf.prefix = "" + ignored_nodes: list[LN] = [] + parent_sibling = parent.prev_sibling + while parent_sibling is not None and parent_sibling.type != syms.suite: + ignored_nodes.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` where the ASYNC token is on the + # grandparent node. + grandparent = parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == token.ASYNC + ): + ignored_nodes.insert(0, grandparent.prev_sibling) + yield from iter(ignored_nodes) + + +def is_fmt_on(container: LN) -> bool: + """Determine whether formatting is switched on within a container. + Determined by whether the last `# fmt:` comment is `on` or `off`. + """ + fmt_on = False + for comment in list_comments(container.prefix, is_endmarker=False): + if comment.value in FMT_ON: + fmt_on = True + elif comment.value in FMT_OFF: + fmt_on = False + return fmt_on + + +def children_contains_fmt_on(container: LN) -> bool: + """Determine if children have formatting switched on.""" + for child in container.children: + leaf = first_leaf_of(child) + if leaf is not None and is_fmt_on(leaf): + return True + + return False + + +def contains_pragma_comment(comment_list: list[Leaf]) -> bool: + """ + Returns: + True iff one of the comments in @comment_list is a pragma used by one + of the more common static analysis tools for python (e.g. mypy, flake8, + pylint). + """ + for comment in comment_list: + if comment.value.startswith(("# type:", "# noqa", "# pylint:")): + return True + + return False + + +def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool: + """ + Checks if the given comment contains FMT_SKIP alone or paired with other comments. + Matching styles: + # fmt:skip <-- single comment + # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview) + # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview) + """ + semantic_comment_blocks = [ + comment_line, + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.split(_COMMENT_PREFIX)[1:] + ], + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.strip(_COMMENT_PREFIX).split( + _COMMENT_LIST_SEPARATOR + ) + ], + ] + + return any(comment in FMT_SKIP for comment in semantic_comment_blocks) diff --git a/contrib/python/black/black/concurrency.py b/contrib/python/black/black/concurrency.py new file mode 100644 index 00000000000..4b3cf48d901 --- /dev/null +++ b/contrib/python/black/black/concurrency.py @@ -0,0 +1,191 @@ +""" +Formatting many files at once via multiprocessing. Contains entrypoint and utilities. + +NOTE: this module is only imported if we need to format several files at once. +""" + +import asyncio +import logging +import os +import signal +import sys +import traceback +from collections.abc import Iterable +from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor +from multiprocessing import Manager +from pathlib import Path +from typing import Any, Optional + +from mypy_extensions import mypyc_attr + +from black import WriteBack, format_file_in_place +from black.cache import Cache +from black.mode import Mode +from black.output import err +from black.report import Changed, Report + + +def maybe_install_uvloop() -> None: + """If our environment has uvloop installed we use it. + + This is called only from command-line entry points to avoid + interfering with the parent process if Black is used as a library. + """ + try: + import uvloop + + uvloop.install() + except ImportError: + pass + + +def cancel(tasks: Iterable["asyncio.Future[Any]"]) -> None: + """asyncio signal handler that cancels all `tasks` and reports to stderr.""" + err("Aborted!") + for task in tasks: + task.cancel() + + +def shutdown(loop: asyncio.AbstractEventLoop) -> None: + """Cancel all pending tasks on `loop`, wait for them, and close the loop.""" + try: + # This part is borrowed from asyncio/runners.py in Python 3.7b2. + to_cancel = [task for task in asyncio.all_tasks(loop) if not task.done()] + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + finally: + # `concurrent.futures.Future` objects cannot be cancelled once they + # are already running. There might be some when the `shutdown()` happened. + # Silence their logger's spew about the event loop being closed. + cf_logger = logging.getLogger("concurrent.futures") + cf_logger.setLevel(logging.CRITICAL) + loop.close() + + +# diff-shades depends on being to monkeypatch this function to operate. I know it's +# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26 +@mypyc_attr(patchable=True) +def reformat_many( + sources: set[Path], + fast: bool, + write_back: WriteBack, + mode: Mode, + report: Report, + workers: Optional[int], +) -> None: + """Reformat multiple files using a ProcessPoolExecutor.""" + maybe_install_uvloop() + + executor: Executor + if workers is None: + workers = int(os.environ.get("BLACK_NUM_WORKERS", 0)) + workers = workers or os.cpu_count() or 1 + if sys.platform == "win32": + # Work around https://bugs.python.org/issue26903 + workers = min(workers, 60) + try: + executor = ProcessPoolExecutor(max_workers=workers) + except (ImportError, NotImplementedError, OSError): + # we arrive here if the underlying system does not support multi-processing + # like in AWS Lambda or Termux, in which case we gracefully fallback to + # a ThreadPoolExecutor with just a single worker (more workers would not do us + # any good due to the Global Interpreter Lock) + executor = ThreadPoolExecutor(max_workers=1) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete( + schedule_formatting( + sources=sources, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + loop=loop, + executor=executor, + ) + ) + finally: + try: + shutdown(loop) + finally: + asyncio.set_event_loop(None) + if executor is not None: + executor.shutdown() + + +async def schedule_formatting( + sources: set[Path], + fast: bool, + write_back: WriteBack, + mode: Mode, + report: "Report", + loop: asyncio.AbstractEventLoop, + executor: "Executor", +) -> None: + """Run formatting of `sources` in parallel using the provided `executor`. + + (Use ProcessPoolExecutors for actual parallelism.) + + `write_back`, `fast`, and `mode` options are passed to + :func:`format_file_in_place`. + """ + cache = Cache.read(mode) + if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): + sources, cached = cache.filtered_cached(sources) + for src in sorted(cached): + report.done(src, Changed.CACHED) + if not sources: + return + + cancelled = [] + sources_to_cache = [] + lock = None + if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): + # For diff output, we need locks to ensure we don't interleave output + # from different processes. + manager = Manager() + lock = manager.Lock() + tasks = { + asyncio.ensure_future( + loop.run_in_executor( + executor, format_file_in_place, src, fast, mode, write_back, lock + ) + ): src + for src in sorted(sources) + } + pending = tasks.keys() + try: + loop.add_signal_handler(signal.SIGINT, cancel, pending) + loop.add_signal_handler(signal.SIGTERM, cancel, pending) + except NotImplementedError: + # There are no good alternatives for these on Windows. + pass + while pending: + done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + for task in done: + src = tasks.pop(task) + if task.cancelled(): + cancelled.append(task) + elif exc := task.exception(): + if report.verbose: + traceback.print_exception(type(exc), exc, exc.__traceback__) + report.failed(src, str(exc)) + else: + changed = Changed.YES if task.result() else Changed.NO + # If the file was written back or was successfully checked as + # well-formatted, store this information in the cache. + if write_back is WriteBack.YES or ( + write_back is WriteBack.CHECK and changed is Changed.NO + ): + sources_to_cache.append(src) + report.done(src, changed) + if cancelled: + await asyncio.gather(*cancelled, return_exceptions=True) + if sources_to_cache: + cache.write(sources_to_cache) diff --git a/contrib/python/black/black/const.py b/contrib/python/black/black/const.py new file mode 100644 index 00000000000..ee466679c70 --- /dev/null +++ b/contrib/python/black/black/const.py @@ -0,0 +1,4 @@ +DEFAULT_LINE_LENGTH = 88 +DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.ipynb_checkpoints|\.mypy_cache|\.nox|\.pytest_cache|\.ruff_cache|\.tox|\.svn|\.venv|\.vscode|__pypackages__|_build|buck-out|build|dist|venv)/" # noqa: B950 +DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" +STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/contrib/python/black/black/debug.py b/contrib/python/black/black/debug.py new file mode 100644 index 00000000000..939b20eee5e --- /dev/null +++ b/contrib/python/black/black/debug.py @@ -0,0 +1,55 @@ +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any, TypeVar, Union + +from black.nodes import Visitor +from black.output import out +from black.parsing import lib2to3_parse +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node, type_repr + +LN = Union[Leaf, Node] +T = TypeVar("T") + + +@dataclass +class DebugVisitor(Visitor[T]): + tree_depth: int = 0 + list_output: list[str] = field(default_factory=list) + print_output: bool = True + + def out(self, message: str, *args: Any, **kwargs: Any) -> None: + self.list_output.append(message) + if self.print_output: + out(message, *args, **kwargs) + + def visit_default(self, node: LN) -> Iterator[T]: + indent = " " * (2 * self.tree_depth) + if isinstance(node, Node): + _type = type_repr(node.type) + self.out(f"{indent}{_type}", fg="yellow") + self.tree_depth += 1 + for child in node.children: + yield from self.visit(child) + + self.tree_depth -= 1 + self.out(f"{indent}/{_type}", fg="yellow", bold=False) + else: + _type = token.tok_name.get(node.type, str(node.type)) + self.out(f"{indent}{_type}", fg="blue", nl=False) + if node.prefix: + # We don't have to handle prefixes for `Node` objects since + # that delegates to the first child anyway. + self.out(f" {node.prefix!r}", fg="green", bold=False, nl=False) + self.out(f" {node.value!r}", fg="blue", bold=False) + + @classmethod + def show(cls, code: Union[str, Leaf, Node]) -> None: + """Pretty-print the lib2to3 AST of a given string of `code`. + + Convenience method for debugging. + """ + v: DebugVisitor[None] = DebugVisitor() + if isinstance(code, str): + code = lib2to3_parse(code) + list(v.visit(code)) diff --git a/contrib/python/black/black/files.py b/contrib/python/black/black/files.py new file mode 100644 index 00000000000..72c5eddf9c0 --- /dev/null +++ b/contrib/python/black/black/files.py @@ -0,0 +1,426 @@ +import io +import os +import sys +from collections.abc import Iterable, Iterator, Sequence +from functools import lru_cache +from pathlib import Path +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, Union + +from mypy_extensions import mypyc_attr +from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet +from packaging.version import InvalidVersion, Version +from pathspec import PathSpec +from pathspec.patterns.gitwildmatch import GitWildMatchPatternError + +if sys.version_info >= (3, 11): + try: + import tomllib + except ImportError: + # Help users on older alphas + if not TYPE_CHECKING: + import tomli as tomllib +else: + import tomli as tomllib + +from black.handle_ipynb_magics import jupyter_dependencies_are_installed +from black.mode import TargetVersion +from black.output import err +from black.report import Report + +if TYPE_CHECKING: + import colorama # noqa: F401 + + +@lru_cache +def _load_toml(path: Union[Path, str]) -> dict[str, Any]: + with open(path, "rb") as f: + return tomllib.load(f) + + +@lru_cache +def _cached_resolve(path: Path) -> Path: + return path.resolve() + + +@lru_cache +def find_project_root( + srcs: Sequence[str], stdin_filename: Optional[str] = None +) -> tuple[Path, str]: + """Return a directory containing .git, .hg, or pyproject.toml. + + pyproject.toml files are only considered if they contain a [tool.black] + section and are ignored otherwise. + + That directory will be a common parent of all files and directories + passed in `srcs`. + + If no directory in the tree contains a marker that would specify it's the + project root, the root of the file system is returned. + + Returns a two-tuple with the first element as the project root path and + the second element as a string describing the method by which the + project root was discovered. + """ + if stdin_filename is not None: + srcs = tuple(stdin_filename if s == "-" else s for s in srcs) + if not srcs: + srcs = [str(_cached_resolve(Path.cwd()))] + + path_srcs = [_cached_resolve(Path(Path.cwd(), src)) for src in srcs] + + # A list of lists of parents for each 'src'. 'src' is included as a + # "parent" of itself if it is a directory + src_parents = [ + list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs + ] + + common_base = max( + set.intersection(*(set(parents) for parents in src_parents)), + key=lambda path: path.parts, + ) + + for directory in (common_base, *common_base.parents): + if (directory / ".git").exists(): + return directory, ".git directory" + + if (directory / ".hg").is_dir(): + return directory, ".hg directory" + + if (directory / "pyproject.toml").is_file(): + pyproject_toml = _load_toml(directory / "pyproject.toml") + if "black" in pyproject_toml.get("tool", {}): + return directory, "pyproject.toml" + + return directory, "file system root" + + +def find_pyproject_toml( + path_search_start: tuple[str, ...], stdin_filename: Optional[str] = None +) -> Optional[str]: + """Find the absolute filepath to a pyproject.toml if it exists""" + path_project_root, _ = find_project_root(path_search_start, stdin_filename) + path_pyproject_toml = path_project_root / "pyproject.toml" + if path_pyproject_toml.is_file(): + return str(path_pyproject_toml) + + try: + path_user_pyproject_toml = find_user_pyproject_toml() + return ( + str(path_user_pyproject_toml) + if path_user_pyproject_toml.is_file() + else None + ) + except (PermissionError, RuntimeError) as e: + # We do not have access to the user-level config directory, so ignore it. + err(f"Ignoring user configuration directory due to {e!r}") + return None + + +@mypyc_attr(patchable=True) +def parse_pyproject_toml(path_config: str) -> dict[str, Any]: + """Parse a pyproject toml file, pulling out relevant parts for Black. + + If parsing fails, will raise a tomllib.TOMLDecodeError. + """ + pyproject_toml = _load_toml(path_config) + config: dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {}) + config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()} + + if "target_version" not in config: + inferred_target_version = infer_target_version(pyproject_toml) + if inferred_target_version is not None: + config["target_version"] = [v.name.lower() for v in inferred_target_version] + + return config + + +def infer_target_version( + pyproject_toml: dict[str, Any], +) -> Optional[list[TargetVersion]]: + """Infer Black's target version from the project metadata in pyproject.toml. + + Supports the PyPA standard format (PEP 621): + https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#requires-python + + If the target version cannot be inferred, returns None. + """ + project_metadata = pyproject_toml.get("project", {}) + requires_python = project_metadata.get("requires-python", None) + if requires_python is not None: + try: + return parse_req_python_version(requires_python) + except InvalidVersion: + pass + try: + return parse_req_python_specifier(requires_python) + except (InvalidSpecifier, InvalidVersion): + pass + + return None + + +def parse_req_python_version(requires_python: str) -> Optional[list[TargetVersion]]: + """Parse a version string (i.e. ``"3.7"``) to a list of TargetVersion. + + If parsing fails, will raise a packaging.version.InvalidVersion error. + If the parsed version cannot be mapped to a valid TargetVersion, returns None. + """ + version = Version(requires_python) + if version.release[0] != 3: + return None + try: + return [TargetVersion(version.release[1])] + except (IndexError, ValueError): + return None + + +def parse_req_python_specifier(requires_python: str) -> Optional[list[TargetVersion]]: + """Parse a specifier string (i.e. ``">=3.7,<3.10"``) to a list of TargetVersion. + + If parsing fails, will raise a packaging.specifiers.InvalidSpecifier error. + If the parsed specifier cannot be mapped to a valid TargetVersion, returns None. + """ + specifier_set = strip_specifier_set(SpecifierSet(requires_python)) + if not specifier_set: + return None + + target_version_map = {f"3.{v.value}": v for v in TargetVersion} + compatible_versions: list[str] = list(specifier_set.filter(target_version_map)) + if compatible_versions: + return [target_version_map[v] for v in compatible_versions] + return None + + +def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet: + """Strip minor versions for some specifiers in the specifier set. + + For background on version specifiers, see PEP 440: + https://peps.python.org/pep-0440/#version-specifiers + """ + specifiers = [] + for s in specifier_set: + if "*" in str(s): + specifiers.append(s) + elif s.operator in ["~=", "==", ">=", "==="]: + version = Version(s.version) + stripped = Specifier(f"{s.operator}{version.major}.{version.minor}") + specifiers.append(stripped) + elif s.operator == ">": + version = Version(s.version) + if len(version.release) > 2: + s = Specifier(f">={version.major}.{version.minor}") + specifiers.append(s) + else: + specifiers.append(s) + + return SpecifierSet(",".join(str(s) for s in specifiers)) + + +@lru_cache +def find_user_pyproject_toml() -> Path: + r"""Return the path to the top-level user configuration for black. + + This looks for ~\.black on Windows and ~/.config/black on Linux and other + Unix systems. + + May raise: + - RuntimeError: if the current user has no homedir + - PermissionError: if the current process cannot access the user's homedir + """ + if sys.platform == "win32": + # Windows + user_config_path = Path.home() / ".black" + else: + config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config") + user_config_path = Path(config_root).expanduser() / "black" + return _cached_resolve(user_config_path) + + +@lru_cache +def get_gitignore(root: Path) -> PathSpec: + """Return a PathSpec matching gitignore content if present.""" + gitignore = root / ".gitignore" + lines: list[str] = [] + if gitignore.is_file(): + with gitignore.open(encoding="utf-8") as gf: + lines = gf.readlines() + try: + return PathSpec.from_lines("gitwildmatch", lines) + except GitWildMatchPatternError as e: + err(f"Could not parse {gitignore}: {e}") + raise + + +def resolves_outside_root_or_cannot_stat( + path: Path, + root: Path, + report: Optional[Report] = None, +) -> bool: + """ + Returns whether the path is a symbolic link that points outside the + root directory. Also returns True if we failed to resolve the path. + """ + try: + resolved_path = _cached_resolve(path) + except OSError as e: + if report: + report.path_ignored(path, f"cannot be read because {e}") + return True + try: + resolved_path.relative_to(root) + except ValueError: + if report: + report.path_ignored(path, f"is a symbolic link that points outside {root}") + return True + return False + + +def best_effort_relative_path(path: Path, root: Path) -> Path: + # Precondition: resolves_outside_root_or_cannot_stat(path, root) is False + try: + return path.absolute().relative_to(root) + except ValueError: + pass + root_parent = next((p for p in path.parents if _cached_resolve(p) == root), None) + if root_parent is not None: + return path.relative_to(root_parent) + # something adversarial, fallback to path guaranteed by precondition + return _cached_resolve(path).relative_to(root) + + +def _path_is_ignored( + root_relative_path: str, + root: Path, + gitignore_dict: dict[Path, PathSpec], +) -> bool: + path = root / root_relative_path + # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must + # ensure that gitignore_dict is ordered from least specific to most specific. + for gitignore_path, pattern in gitignore_dict.items(): + try: + relative_path = path.relative_to(gitignore_path).as_posix() + if path.is_dir(): + relative_path = relative_path + "/" + except ValueError: + break + if pattern.match_file(relative_path): + return True + return False + + +def path_is_excluded( + normalized_path: str, + pattern: Optional[Pattern[str]], +) -> bool: + match = pattern.search(normalized_path) if pattern else None + return bool(match and match.group(0)) + + +def gen_python_files( + paths: Iterable[Path], + root: Path, + include: Pattern[str], + exclude: Pattern[str], + extend_exclude: Optional[Pattern[str]], + force_exclude: Optional[Pattern[str]], + report: Report, + gitignore_dict: Optional[dict[Path, PathSpec]], + *, + verbose: bool, + quiet: bool, +) -> Iterator[Path]: + """Generate all files under `path` whose paths are not excluded by the + `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, + but are included by the `include` regex. + + Symbolic links pointing outside of the `root` directory are ignored. + + `report` is where output about exclusions goes. + """ + + assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" + for child in paths: + assert child.is_absolute() + root_relative_path = child.relative_to(root).as_posix() + + # First ignore files matching .gitignore, if passed + if gitignore_dict and _path_is_ignored( + root_relative_path, root, gitignore_dict + ): + report.path_ignored(child, "matches a .gitignore file content") + continue + + # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. + root_relative_path = "/" + root_relative_path + if child.is_dir(): + root_relative_path += "/" + + if path_is_excluded(root_relative_path, exclude): + report.path_ignored(child, "matches the --exclude regular expression") + continue + + if path_is_excluded(root_relative_path, extend_exclude): + report.path_ignored( + child, "matches the --extend-exclude regular expression" + ) + continue + + if path_is_excluded(root_relative_path, force_exclude): + report.path_ignored(child, "matches the --force-exclude regular expression") + continue + + if resolves_outside_root_or_cannot_stat(child, root, report): + continue + + if child.is_dir(): + # If gitignore is None, gitignore usage is disabled, while a Falsey + # gitignore is when the directory doesn't have a .gitignore file. + if gitignore_dict is not None: + new_gitignore_dict = { + **gitignore_dict, + root / child: get_gitignore(child), + } + else: + new_gitignore_dict = None + yield from gen_python_files( + child.iterdir(), + root, + include, + exclude, + extend_exclude, + force_exclude, + report, + new_gitignore_dict, + verbose=verbose, + quiet=quiet, + ) + + elif child.is_file(): + if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + warn=verbose or not quiet + ): + continue + include_match = include.search(root_relative_path) if include else True + if include_match: + yield child + + +def wrap_stream_for_windows( + f: io.TextIOWrapper, +) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]: + """ + Wrap stream with colorama's wrap_stream so colors are shown on Windows. + + If `colorama` is unavailable, the original stream is returned unmodified. + Otherwise, the `wrap_stream()` function determines whether the stream needs + to be wrapped for a Windows environment and will accordingly either return + an `AnsiToWin32` wrapper or the original stream. + """ + try: + from colorama.initialise import wrap_stream + except ImportError: + return f + else: + # Set `strip=False` to avoid needing to modify test_express_diff_with_color. + return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True) diff --git a/contrib/python/black/black/handle_ipynb_magics.py b/contrib/python/black/black/handle_ipynb_magics.py new file mode 100644 index 00000000000..dd680bffffb --- /dev/null +++ b/contrib/python/black/black/handle_ipynb_magics.py @@ -0,0 +1,508 @@ +"""Functions to process IPython magics with.""" + +import ast +import collections +import dataclasses +import re +import secrets +import sys +from functools import lru_cache +from importlib.util import find_spec +from typing import Optional + +if sys.version_info >= (3, 10): + from typing import TypeGuard +else: + from typing_extensions import TypeGuard + +from black.mode import Mode +from black.output import out +from black.report import NothingChanged + +TRANSFORMED_MAGICS = frozenset(( + "get_ipython().run_cell_magic", + "get_ipython().system", + "get_ipython().getoutput", + "get_ipython().run_line_magic", +)) +TOKENS_TO_IGNORE = frozenset(( + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", +)) +PYTHON_CELL_MAGICS = frozenset(( + "capture", + "prun", + "pypy", + "python", + "python3", + "time", + "timeit", +)) + + [email protected](frozen=True) +class Replacement: + mask: str + src: str + + +@lru_cache +def jupyter_dependencies_are_installed(*, warn: bool) -> bool: + installed = ( + find_spec("tokenize_rt") is not None and find_spec("IPython") is not None + ) + if not installed and warn: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + 'You can fix this by running ``pip install "black[jupyter]"``' + ) + out(msg) + return installed + + +def validate_cell(src: str, mode: Mode) -> None: + """Check that cell does not already contain TransformerManager transformations, + or non-Python cell magics, which might cause tokenizer_rt to break because of + indentations. + + If a cell contains ``!ls``, then it'll be transformed to + ``get_ipython().system('ls')``. However, if the cell originally contained + ``get_ipython().system('ls')``, then it would get transformed in the same way: + + >>> TransformerManager().transform_cell("get_ipython().system('ls')") + "get_ipython().system('ls')\n" + >>> TransformerManager().transform_cell("!ls") + "get_ipython().system('ls')\n" + + Due to the impossibility of safely roundtripping in such situations, cells + containing transformed magics will be ignored. + """ + if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS): + raise NothingChanged + + line = _get_code_start(src) + if line.startswith("%%") and ( + line.split(maxsplit=1)[0][2:] + not in PYTHON_CELL_MAGICS | mode.python_cell_magics + ): + raise NothingChanged + + +def remove_trailing_semicolon(src: str) -> tuple[str, bool]: + """Remove trailing semicolon from Jupyter notebook cell. + + For example, + + fig, ax = plt.subplots() + ax.plot(x_data, y_data); # plot data + + would become + + fig, ax = plt.subplots() + ax.plot(x_data, y_data) # plot data + + Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses + ``tokenize_rt`` so that round-tripping works fine. + """ + from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src + + tokens = src_to_tokens(src) + trailing_semicolon = False + for idx, token in reversed_enumerate(tokens): + if token.name in TOKENS_TO_IGNORE: + continue + if token.name == "OP" and token.src == ";": + del tokens[idx] + trailing_semicolon = True + break + if not trailing_semicolon: + return src, False + return tokens_to_src(tokens), True + + +def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: + """Put trailing semicolon back if cell originally had it. + + Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses + ``tokenize_rt`` so that round-tripping works fine. + """ + if not has_trailing_semicolon: + return src + from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src + + tokens = src_to_tokens(src) + for idx, token in reversed_enumerate(tokens): + if token.name in TOKENS_TO_IGNORE: + continue + tokens[idx] = token._replace(src=token.src + ";") + break + else: # pragma: nocover + raise AssertionError( + "INTERNAL ERROR: Was not able to reinstate trailing semicolon. " + "Please report a bug on https://github.com/psf/black/issues. " + ) from None + return str(tokens_to_src(tokens)) + + +def mask_cell(src: str) -> tuple[str, list[Replacement]]: + """Mask IPython magics so content becomes parseable Python code. + + For example, + + %matplotlib inline + 'foo' + + becomes + + b"25716f358c32750" + 'foo' + + The replacements are returned, along with the transformed code. + """ + replacements: list[Replacement] = [] + try: + ast.parse(src) + except SyntaxError: + # Might have IPython magics, will process below. + pass + else: + # Syntax is fine, nothing to mask, early return. + return src, replacements + + from IPython.core.inputtransformer2 import TransformerManager + + transformer_manager = TransformerManager() + # A side effect of the following transformation is that it also removes any + # empty lines at the beginning of the cell. + transformed = transformer_manager.transform_cell(src) + transformed, cell_magic_replacements = replace_cell_magics(transformed) + replacements += cell_magic_replacements + transformed = transformer_manager.transform_cell(transformed) + transformed, magic_replacements = replace_magics(transformed) + if len(transformed.strip().splitlines()) != len(src.strip().splitlines()): + # Multi-line magic, not supported. + raise NothingChanged + replacements += magic_replacements + return transformed, replacements + + +def create_token(n_chars: int) -> str: + """Create a randomly generated token that is n_chars characters long.""" + assert n_chars > 0 + n_bytes = max(n_chars // 2 - 1, 1) + token = secrets.token_hex(n_bytes) + if len(token) + 3 > n_chars: + token = token[:-1] + # We use a bytestring so that the string does not get interpreted + # as a docstring. + return f'b"{token}"' + + +def get_token(src: str, magic: str) -> str: + """Return randomly generated token to mask IPython magic with. + + For example, if 'magic' was `%matplotlib inline`, then a possible + token to mask it with would be `"43fdd17f7e5ddc83"`. The token + will be the same length as the magic, and we make sure that it was + not already present anywhere else in the cell. + """ + assert magic + n_chars = len(magic) + token = create_token(n_chars) + counter = 0 + while token in src: + token = create_token(n_chars) + counter += 1 + if counter > 100: + raise AssertionError( + "INTERNAL ERROR: Black was not able to replace IPython magic. " + "Please report a bug on https://github.com/psf/black/issues. " + f"The magic might be helpful: {magic}" + ) from None + return token + + +def replace_cell_magics(src: str) -> tuple[str, list[Replacement]]: + """Replace cell magic with token. + + Note that 'src' will already have been processed by IPython's + TransformerManager().transform_cell. + + Example, + + get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\\n') + + becomes + + "a794." + ls =!ls + + The replacement, along with the transformed code, is returned. + """ + replacements: list[Replacement] = [] + + tree = ast.parse(src) + + cell_magic_finder = CellMagicFinder() + cell_magic_finder.visit(tree) + if cell_magic_finder.cell_magic is None: + return src, replacements + header = cell_magic_finder.cell_magic.header + mask = get_token(src, header) + replacements.append(Replacement(mask=mask, src=header)) + return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements + + +def replace_magics(src: str) -> tuple[str, list[Replacement]]: + """Replace magics within body of cell. + + Note that 'src' will already have been processed by IPython's + TransformerManager().transform_cell. + + Example, this + + get_ipython().run_line_magic('matplotlib', 'inline') + 'foo' + + becomes + + "5e67db56d490fd39" + 'foo' + + The replacement, along with the transformed code, are returned. + """ + replacements = [] + magic_finder = MagicFinder() + magic_finder.visit(ast.parse(src)) + new_srcs = [] + for i, line in enumerate(src.split("\n"), start=1): + if i in magic_finder.magics: + offsets_and_magics = magic_finder.magics[i] + if len(offsets_and_magics) != 1: # pragma: nocover + raise AssertionError( + f"Expecting one magic per line, got: {offsets_and_magics}\n" + "Please report a bug on https://github.com/psf/black/issues." + ) + col_offset, magic = ( + offsets_and_magics[0].col_offset, + offsets_and_magics[0].magic, + ) + mask = get_token(src, magic) + replacements.append(Replacement(mask=mask, src=magic)) + line = line[:col_offset] + mask + new_srcs.append(line) + return "\n".join(new_srcs), replacements + + +def unmask_cell(src: str, replacements: list[Replacement]) -> str: + """Remove replacements from cell. + + For example + + "9b20" + foo = bar + + becomes + + %%time + foo = bar + """ + for replacement in replacements: + src = src.replace(replacement.mask, replacement.src) + return src + + +def _get_code_start(src: str) -> str: + """Provides the first line where the code starts. + + Iterates over lines of code until it finds the first line that doesn't + contain only empty spaces and comments. It removes any empty spaces at the + start of the line and returns it. If such line doesn't exist, it returns an + empty string. + """ + for match in re.finditer(".+", src): + line = match.group(0).lstrip() + if line and not line.startswith("#"): + return line + return "" + + +def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]: + """Check if attribute is IPython magic. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + """ + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "get_ipython" + ) + + +def _get_str_args(args: list[ast.expr]) -> list[str]: + str_args = [] + for arg in args: + assert isinstance(arg, ast.Constant) and isinstance(arg.value, str) + str_args.append(arg.value) + return str_args + + [email protected](frozen=True) +class CellMagic: + name: str + params: Optional[str] + body: str + + @property + def header(self) -> str: + if self.params: + return f"%%{self.name} {self.params}" + return f"%%{self.name}" + + +# ast.NodeVisitor + dataclass = breakage under mypyc. +class CellMagicFinder(ast.NodeVisitor): + """Find cell magics. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + + For example, + + %%time\n + foo() + + would have been transformed to + + get_ipython().run_cell_magic('time', '', 'foo()\\n') + + and we look for instances of the latter. + """ + + def __init__(self, cell_magic: Optional[CellMagic] = None) -> None: + self.cell_magic = cell_magic + + def visit_Expr(self, node: ast.Expr) -> None: + """Find cell magic, extract header and body.""" + if ( + isinstance(node.value, ast.Call) + and _is_ipython_magic(node.value.func) + and node.value.func.attr == "run_cell_magic" + ): + args = _get_str_args(node.value.args) + self.cell_magic = CellMagic(name=args[0], params=args[1], body=args[2]) + self.generic_visit(node) + + [email protected](frozen=True) +class OffsetAndMagic: + col_offset: int + magic: str + + +# Unsurprisingly, subclassing ast.NodeVisitor means we can't use dataclasses here +# as mypyc will generate broken code. +class MagicFinder(ast.NodeVisitor): + """Visit cell to look for get_ipython calls. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + + For example, + + %matplotlib inline + + would have been transformed to + + get_ipython().run_line_magic('matplotlib', 'inline') + + and we look for instances of the latter (and likewise for other + types of magics). + """ + + def __init__(self) -> None: + self.magics: dict[int, list[OffsetAndMagic]] = collections.defaultdict(list) + + def visit_Assign(self, node: ast.Assign) -> None: + """Look for system assign magics. + + For example, + + black_version = !black --version + env = %env var + + would have been (respectively) transformed to + + black_version = get_ipython().getoutput('black --version') + env = get_ipython().run_line_magic('env', 'var') + + and we look for instances of any of the latter. + """ + if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): + args = _get_str_args(node.value.args) + if node.value.func.attr == "getoutput": + src = f"!{args[0]}" + elif node.value.func.attr == "run_line_magic": + src = f"%{args[0]}" + if args[1]: + src += f" {args[1]}" + else: + raise AssertionError( + f"Unexpected IPython magic {node.value.func.attr!r} found. " + "Please report a bug on https://github.com/psf/black/issues." + ) from None + self.magics[node.value.lineno].append( + OffsetAndMagic(node.value.col_offset, src) + ) + self.generic_visit(node) + + def visit_Expr(self, node: ast.Expr) -> None: + """Look for magics in body of cell. + + For examples, + + !ls + !!ls + ?ls + ??ls + + would (respectively) get transformed to + + get_ipython().system('ls') + get_ipython().getoutput('ls') + get_ipython().run_line_magic('pinfo', 'ls') + get_ipython().run_line_magic('pinfo2', 'ls') + + and we look for instances of any of the latter. + """ + if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): + args = _get_str_args(node.value.args) + if node.value.func.attr == "run_line_magic": + if args[0] == "pinfo": + src = f"?{args[1]}" + elif args[0] == "pinfo2": + src = f"??{args[1]}" + else: + src = f"%{args[0]}" + if args[1]: + src += f" {args[1]}" + elif node.value.func.attr == "system": + src = f"!{args[0]}" + elif node.value.func.attr == "getoutput": + src = f"!!{args[0]}" + else: + raise NothingChanged # unsupported magic. + self.magics[node.value.lineno].append( + OffsetAndMagic(node.value.col_offset, src) + ) + self.generic_visit(node) diff --git a/contrib/python/black/black/linegen.py b/contrib/python/black/black/linegen.py new file mode 100644 index 00000000000..ee65a7a6e40 --- /dev/null +++ b/contrib/python/black/black/linegen.py @@ -0,0 +1,1844 @@ +""" +Generating lines of code. +""" + +import re +import sys +from collections.abc import Collection, Iterator +from dataclasses import replace +from enum import Enum, auto +from functools import partial, wraps +from typing import Optional, Union, cast + +from black.brackets import ( + COMMA_PRIORITY, + DOT_PRIORITY, + STRING_PRIORITY, + get_leaves_inside_matching_brackets, + max_delimiter_priority_in_atom, +) +from black.comments import FMT_OFF, generate_comments, list_comments +from black.lines import ( + Line, + RHSResult, + append_leaves, + can_be_split, + can_omit_invisible_parens, + is_line_short_enough, + line_to_string, +) +from black.mode import Feature, Mode, Preview +from black.nodes import ( + ASSIGNMENTS, + BRACKETS, + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + STATEMENT, + WHITESPACE, + Visitor, + ensure_visible, + fstring_to_string, + get_annotation_type, + is_arith_like, + is_async_stmt_or_funcdef, + is_atom_with_invisible_parens, + is_docstring, + is_empty_tuple, + is_generator, + is_lpar_token, + is_multiline_string, + is_name_token, + is_one_sequence_between, + is_one_tuple, + is_parent_function_or_class, + is_part_of_annotation, + is_rpar_token, + is_stub_body, + is_stub_suite, + is_tuple_containing_star, + is_tuple_containing_walrus, + is_type_ignore_comment_string, + is_vararg, + is_walrus_assignment, + is_yield, + syms, + wrap_in_parentheses, +) +from black.numerics import normalize_numeric_literal +from black.strings import ( + fix_multiline_docstring, + get_string_prefix, + normalize_string_prefix, + normalize_string_quotes, + normalize_unicode_escape_sequences, +) +from black.trans import ( + CannotTransform, + StringMerger, + StringParenStripper, + StringParenWrapper, + StringSplitter, + Transformer, + hug_power_op, +) +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node + +# types +LeafID = int +LN = Union[Leaf, Node] + + +class CannotSplit(CannotTransform): + """A readable split that fits the allotted line length is impossible.""" + + +# This isn't a dataclass because @dataclass + Generic breaks mypyc. +# See also https://github.com/mypyc/mypyc/issues/827. +class LineGenerator(Visitor[Line]): + """Generates reformatted Line objects. Empty lines are not emitted. + + Note: destroys the tree it's visiting by mutating prefixes of its leaves + in ways that will no longer stringify to valid Python code on the tree. + """ + + def __init__(self, mode: Mode, features: Collection[Feature]) -> None: + self.mode = mode + self.features = features + self.current_line: Line + self.__post_init__() + + def line(self, indent: int = 0) -> Iterator[Line]: + """Generate a line. + + If the line is empty, only emit if it makes sense. + If the line is too long, split it first and then generate. + + If any lines were generated, set up a new current_line. + """ + if not self.current_line: + self.current_line.depth += indent + return # Line is empty, don't emit. Creating a new one unnecessary. + + if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef( + self.current_line.leaves[0] + ): + # Special case for async def/for/with statements. `visit_async_stmt` + # adds an `ASYNC` leaf then visits the child def/for/with statement + # nodes. Line yields from those nodes shouldn't treat the former + # `ASYNC` leaf as a complete line. + return + + complete_line = self.current_line + self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent) + yield complete_line + + def visit_default(self, node: LN) -> Iterator[Line]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" + if isinstance(node, Leaf): + any_open_brackets = self.current_line.bracket_tracker.any_open_brackets() + for comment in generate_comments(node): + if any_open_brackets: + # any comment within brackets is subject to splitting + self.current_line.append(comment) + elif comment.type == token.COMMENT: + # regular trailing comment + self.current_line.append(comment) + yield from self.line() + + else: + # regular standalone comment + yield from self.line() + + self.current_line.append(comment) + yield from self.line() + + if any_open_brackets: + node.prefix = "" + if node.type not in WHITESPACE: + self.current_line.append(node) + yield from super().visit_default(node) + + def visit_test(self, node: Node) -> Iterator[Line]: + """Visit an `x if y else z` test""" + + already_parenthesized = ( + node.prev_sibling and node.prev_sibling.type == token.LPAR + ) + + if not already_parenthesized: + # Similar to logic in wrap_in_parentheses + lpar = Leaf(token.LPAR, "") + rpar = Leaf(token.RPAR, "") + prefix = node.prefix + node.prefix = "" + lpar.prefix = prefix + node.insert_child(0, lpar) + node.append_child(rpar) + + yield from self.visit_default(node) + + def visit_INDENT(self, node: Leaf) -> Iterator[Line]: + """Increase indentation level, maybe yield a line.""" + # In blib2to3 INDENT never holds comments. + yield from self.line(+1) + yield from self.visit_default(node) + + def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: + """Decrease indentation level, maybe yield a line.""" + # The current line might still wait for trailing comments. At DEDENT time + # there won't be any (they would be prefixes on the preceding NEWLINE). + # Emit the line then. + yield from self.line() + + # While DEDENT has no value, its prefix may contain standalone comments + # that belong to the current indentation level. Get 'em. + yield from self.visit_default(node) + + # Finally, emit the dedent. + yield from self.line(-1) + + def visit_stmt( + self, node: Node, keywords: set[str], parens: set[str] + ) -> Iterator[Line]: + """Visit a statement. + + This implementation is shared for `if`, `while`, `for`, `try`, `except`, + `def`, `with`, `class`, `assert`, and assignments. + + The relevant Python language `keywords` for a given statement will be + NAME leaves within it. This methods puts those on a separate line. + + `parens` holds a set of string leaf values immediately after which + invisible parens should be put. + """ + normalize_invisible_parens( + node, parens_after=parens, mode=self.mode, features=self.features + ) + for child in node.children: + if is_name_token(child) and child.value in keywords: + yield from self.line() + + yield from self.visit(child) + + def visit_typeparams(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[0].prefix = "" + + def visit_typevartuple(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + + def visit_paramspec(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + + def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: + if Preview.wrap_long_dict_values_in_parens in self.mode: + for i, child in enumerate(node.children): + if i == 0: + continue + if node.children[i - 1].type == token.COLON: + if ( + child.type == syms.atom + and child.children[0].type in OPENING_BRACKETS + and not is_walrus_assignment(child) + ): + maybe_make_parens_invisible_in_atom( + child, + parent=node, + remove_brackets_around_comma=False, + ) + else: + wrap_in_parentheses(node, child, visible=False) + yield from self.visit_default(node) + + def visit_funcdef(self, node: Node) -> Iterator[Line]: + """Visit function definition.""" + yield from self.line() + + # Remove redundant brackets around return type annotation. + is_return_annotation = False + for child in node.children: + if child.type == token.RARROW: + is_return_annotation = True + elif is_return_annotation: + if child.type == syms.atom and child.children[0].type == token.LPAR: + if maybe_make_parens_invisible_in_atom( + child, + parent=node, + remove_brackets_around_comma=False, + ): + wrap_in_parentheses(node, child, visible=False) + else: + wrap_in_parentheses(node, child, visible=False) + is_return_annotation = False + + for child in node.children: + yield from self.visit(child) + + def visit_match_case(self, node: Node) -> Iterator[Line]: + """Visit either a match or case statement.""" + normalize_invisible_parens( + node, parens_after=set(), mode=self.mode, features=self.features + ) + + yield from self.line() + for child in node.children: + yield from self.visit(child) + + def visit_suite(self, node: Node) -> Iterator[Line]: + """Visit a suite.""" + if is_stub_suite(node): + yield from self.visit(node.children[2]) + else: + yield from self.visit_default(node) + + def visit_simple_stmt(self, node: Node) -> Iterator[Line]: + """Visit a statement without nested statements.""" + prev_type: Optional[int] = None + for child in node.children: + if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child): + wrap_in_parentheses(node, child, visible=False) + prev_type = child.type + + if node.parent and node.parent.type in STATEMENT: + if is_parent_function_or_class(node) and is_stub_body(node): + yield from self.visit_default(node) + else: + yield from self.line(+1) + yield from self.visit_default(node) + yield from self.line(-1) + + else: + if node.parent and is_stub_suite(node.parent): + node.prefix = "" + yield from self.visit_default(node) + return + yield from self.line() + yield from self.visit_default(node) + + def visit_async_stmt(self, node: Node) -> Iterator[Line]: + """Visit `async def`, `async for`, `async with`.""" + yield from self.line() + + children = iter(node.children) + for child in children: + yield from self.visit(child) + + if child.type == token.ASYNC or child.type == STANDALONE_COMMENT: + # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async + # line. + break + + internal_stmt = next(children) + yield from self.visit(internal_stmt) + + def visit_decorators(self, node: Node) -> Iterator[Line]: + """Visit decorators.""" + for child in node.children: + yield from self.line() + yield from self.visit(child) + + def visit_power(self, node: Node) -> Iterator[Line]: + for idx, leaf in enumerate(node.children[:-1]): + next_leaf = node.children[idx + 1] + + if not isinstance(leaf, Leaf): + continue + + value = leaf.value.lower() + if ( + leaf.type == token.NUMBER + and next_leaf.type == syms.trailer + # Ensure that we are in an attribute trailer + and next_leaf.children[0].type == token.DOT + # It shouldn't wrap hexadecimal, binary and octal literals + and not value.startswith(("0x", "0b", "0o")) + # It shouldn't wrap complex literals + and "j" not in value + ): + wrap_in_parentheses(node, leaf) + + remove_await_parens(node) + + yield from self.visit_default(node) + + def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]: + """Remove a semicolon and put the other statement on a separate line.""" + yield from self.line() + + def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]: + """End of file. Process outstanding comments and end with a newline.""" + yield from self.visit_default(leaf) + yield from self.line() + + def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: + if not self.current_line.bracket_tracker.any_open_brackets(): + yield from self.line() + yield from self.visit_default(leaf) + + def visit_factor(self, node: Node) -> Iterator[Line]: + """Force parentheses between a unary op and a binary power: + + -2 ** 8 -> -(2 ** 8) + """ + _operator, operand = node.children + if ( + operand.type == syms.power + and len(operand.children) == 3 + and operand.children[1].type == token.DOUBLESTAR + ): + lpar = Leaf(token.LPAR, "(") + rpar = Leaf(token.RPAR, ")") + index = operand.remove() or 0 + node.insert_child(index, Node(syms.atom, [lpar, operand, rpar])) + yield from self.visit_default(node) + + def visit_tname(self, node: Node) -> Iterator[Line]: + """ + Add potential parentheses around types in function parameter lists to be made + into real parentheses in case the type hint is too long to fit on a line + Examples: + def foo(a: int, b: float = 7): ... + + -> + + def foo(a: (int), b: (float) = 7): ... + """ + assert len(node.children) == 3 + if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + wrap_in_parentheses(node, node.children[2], visible=False) + + yield from self.visit_default(node) + + def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: + normalize_unicode_escape_sequences(leaf) + + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): + # We're ignoring docstrings with backslash newline escapes because changing + # indentation of those changes the AST representation of the code. + if self.mode.string_normalization: + docstring = normalize_string_prefix(leaf.value) + # We handle string normalization at the end of this method, but since + # what we do right now acts differently depending on quote style (ex. + # see padding logic below), there's a possibility for unstable + # formatting. To avoid a situation where this function formats a + # docstring differently on the second pass, normalize it early. + docstring = normalize_string_quotes(docstring) + else: + docstring = leaf.value + prefix = get_string_prefix(docstring) + docstring = docstring[len(prefix) :] # Remove the prefix + quote_char = docstring[0] + # A natural way to remove the outer quotes is to do: + # docstring = docstring.strip(quote_char) + # but that breaks on """""x""" (which is '""x'). + # So we actually need to remove the first character and the next two + # characters but only if they are the same as the first. + quote_len = 1 if docstring[1] != quote_char else 3 + docstring = docstring[quote_len:-quote_len] + docstring_started_empty = not docstring + indent = " " * 4 * self.current_line.depth + + if is_multiline_string(leaf): + docstring = fix_multiline_docstring(docstring, indent) + else: + docstring = docstring.strip() + + has_trailing_backslash = False + if docstring: + # Add some padding if the docstring starts / ends with a quote mark. + if docstring[0] == quote_char: + docstring = " " + docstring + if docstring[-1] == quote_char: + docstring += " " + if docstring[-1] == "\\": + backslash_count = len(docstring) - len(docstring.rstrip("\\")) + if backslash_count % 2: + # Odd number of tailing backslashes, add some padding to + # avoid escaping the closing string quote. + docstring += " " + has_trailing_backslash = True + elif not docstring_started_empty: + docstring = " " + + # We could enforce triple quotes at this point. + quote = quote_char * quote_len + + # It's invalid to put closing single-character quotes on a new line. + if quote_len == 3: + # We need to find the length of the last line of the docstring + # to find if we can add the closing quotes to the line without + # exceeding the maximum line length. + # If docstring is one line, we don't put the closing quotes on a + # separate line because it looks ugly (#3320). + lines = docstring.splitlines() + last_line_length = len(lines[-1]) if docstring else 0 + + # If adding closing quotes would cause the last line to exceed + # the maximum line length, and the closing quote is not + # prefixed by a newline then put a line break before + # the closing quotes + if ( + len(lines) > 1 + and last_line_length + quote_len > self.mode.line_length + and len(indent) + quote_len <= self.mode.line_length + and not has_trailing_backslash + ): + if leaf.value[-1 - quote_len] == "\n": + leaf.value = prefix + quote + docstring + quote + else: + leaf.value = prefix + quote + docstring + "\n" + indent + quote + else: + leaf.value = prefix + quote + docstring + quote + else: + leaf.value = prefix + quote + docstring + quote + + if self.mode.string_normalization and leaf.type == token.STRING: + leaf.value = normalize_string_prefix(leaf.value) + leaf.value = normalize_string_quotes(leaf.value) + yield from self.visit_default(leaf) + + def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: + normalize_numeric_literal(leaf) + yield from self.visit_default(leaf) + + def visit_atom(self, node: Node) -> Iterator[Line]: + """Visit any atom""" + if len(node.children) == 3: + first = node.children[0] + last = node.children[-1] + if (first.type == token.LSQB and last.type == token.RSQB) or ( + first.type == token.LBRACE and last.type == token.RBRACE + ): + # Lists or sets of one item + maybe_make_parens_invisible_in_atom(node.children[1], parent=node) + + yield from self.visit_default(node) + + def visit_fstring(self, node: Node) -> Iterator[Line]: + # currently we don't want to format and split f-strings at all. + string_leaf = fstring_to_string(node) + node.replace(string_leaf) + if "\\" in string_leaf.value and any( + "\\" in str(child) + for child in node.children + if child.type == syms.fstring_replacement_field + ): + # string normalization doesn't account for nested quotes, + # causing breakages. skip normalization when nested quotes exist + yield from self.visit_default(string_leaf) + return + yield from self.visit_STRING(string_leaf) + + # TODO: Uncomment Implementation to format f-string children + # fstring_start = node.children[0] + # fstring_end = node.children[-1] + # assert isinstance(fstring_start, Leaf) + # assert isinstance(fstring_end, Leaf) + + # quote_char = fstring_end.value[0] + # quote_idx = fstring_start.value.index(quote_char) + # prefix, quote = ( + # fstring_start.value[:quote_idx], + # fstring_start.value[quote_idx:] + # ) + + # if not is_docstring(node, self.mode): + # prefix = normalize_string_prefix(prefix) + + # assert quote == fstring_end.value + + # is_raw_fstring = "r" in prefix or "R" in prefix + # middles = [ + # leaf + # for leaf in node.leaves() + # if leaf.type == token.FSTRING_MIDDLE + # ] + + # if self.mode.string_normalization: + # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring) + + # fstring_start.value = prefix + quote + # fstring_end.value = quote + + # yield from self.visit_default(node) + + def __post_init__(self) -> None: + """You are in a twisty little maze of passages.""" + self.current_line = Line(mode=self.mode) + + v = self.visit_stmt + Ø: set[str] = set() + self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","}) + self.visit_if_stmt = partial( + v, keywords={"if", "else", "elif"}, parens={"if", "elif"} + ) + self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"}) + self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"}) + self.visit_try_stmt = partial( + v, keywords={"try", "except", "else", "finally"}, parens=Ø + ) + self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"}) + self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"}) + self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) + + self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) + self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) + self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) + self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) + self.visit_async_funcdef = self.visit_async_stmt + self.visit_decorated = self.visit_decorators + + # PEP 634 + self.visit_match_stmt = self.visit_match_case + self.visit_case_block = self.visit_match_case + self.visit_guard = partial(v, keywords=Ø, parens={"if"}) + + +def _hugging_power_ops_line_to_string( + line: Line, + features: Collection[Feature], + mode: Mode, +) -> Optional[str]: + try: + return line_to_string(next(hug_power_op(line, features, mode))) + except CannotTransform: + return None + + +def transform_line( + line: Line, mode: Mode, features: Collection[Feature] = () +) -> Iterator[Line]: + """Transform a `line`, potentially splitting it into many lines. + + They should fit in the allotted `line_length` but might not be able to. + + `features` are syntactical features that may be used in the output. + """ + if line.is_comment: + yield line + return + + line_str = line_to_string(line) + + # We need the line string when power operators are hugging to determine if we should + # split the line. Default to line_str, if no power operator are present on the line. + line_str_hugging_power_ops = ( + _hugging_power_ops_line_to_string(line, features, mode) or line_str + ) + + ll = mode.line_length + sn = mode.string_normalization + string_merge = StringMerger(ll, sn) + string_paren_strip = StringParenStripper(ll, sn) + string_split = StringSplitter(ll, sn) + string_paren_wrap = StringParenWrapper(ll, sn) + + transformers: list[Transformer] + if ( + not line.contains_uncollapsable_type_comments() + and not line.should_split_rhs + and not line.magic_trailing_comma + and ( + is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops) + or line.contains_unsplittable_type_ignore() + ) + and not (line.inside_brackets and line.contains_standalone_comments()) + and not line.contains_implicit_multiline_string_with_comments() + ): + # Only apply basic string preprocessing, since lines shouldn't be split here. + if Preview.string_processing in mode: + transformers = [string_merge, string_paren_strip] + else: + transformers = [] + elif line.is_def and not should_split_funcdef_with_rhs(line, mode): + transformers = [left_hand_split] + else: + + def _rhs( + self: object, line: Line, features: Collection[Feature], mode: Mode + ) -> Iterator[Line]: + """Wraps calls to `right_hand_split`. + + The calls increasingly `omit` right-hand trailers (bracket pairs with + content), meaning the trailers get glued together to split on another + bracket pair instead. + """ + for omit in generate_trailers_to_omit(line, mode.line_length): + lines = list(right_hand_split(line, mode, features, omit=omit)) + # Note: this check is only able to figure out if the first line of the + # *current* transformation fits in the line length. This is true only + # for simple cases. All others require running more transforms via + # `transform_line()`. This check doesn't know if those would succeed. + if is_line_short_enough(lines[0], mode=mode): + yield from lines + return + + # All splits failed, best effort split with no omits. + # This mostly happens to multiline strings that are by definition + # reported as not fitting a single line, as well as lines that contain + # trailing commas (those have to be exploded). + yield from right_hand_split(line, mode, features=features) + + # HACK: nested functions (like _rhs) compiled by mypyc don't retain their + # __name__ attribute which is needed in `run_transformer` further down. + # Unfortunately a nested class breaks mypyc too. So a class must be created + # via type ... https://github.com/mypyc/mypyc/issues/884 + rhs = type("rhs", (), {"__call__": _rhs})() + + if Preview.string_processing in mode: + if line.inside_brackets: + transformers = [ + string_merge, + string_paren_strip, + string_split, + delimiter_split, + standalone_comment_split, + string_paren_wrap, + rhs, + ] + else: + transformers = [ + string_merge, + string_paren_strip, + string_split, + string_paren_wrap, + rhs, + ] + else: + if line.inside_brackets: + transformers = [delimiter_split, standalone_comment_split, rhs] + else: + transformers = [rhs] + # It's always safe to attempt hugging of power operations and pretty much every line + # could match. + transformers.append(hug_power_op) + + for transform in transformers: + # We are accumulating lines in `result` because we might want to abort + # mission and return the original line in the end, or attempt a different + # split altogether. + try: + result = run_transformer(line, transform, mode, features, line_str=line_str) + except CannotTransform: + continue + else: + yield from result + break + + else: + yield line + + +def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool: + """If a funcdef has a magic trailing comma in the return type, then we should first + split the line with rhs to respect the comma. + """ + return_type_leaves: list[Leaf] = [] + in_return_type = False + + for leaf in line.leaves: + if leaf.type == token.COLON: + in_return_type = False + if in_return_type: + return_type_leaves.append(leaf) + if leaf.type == token.RARROW: + in_return_type = True + + # using `bracket_split_build_line` will mess with whitespace, so we duplicate a + # couple lines from it. + result = Line(mode=line.mode, depth=line.depth) + leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves) + for leaf in return_type_leaves: + result.append( + leaf, + preformatted=True, + track_bracket=id(leaf) in leaves_to_track, + ) + + # we could also return true if the line is too long, and the return type is longer + # than the param list. Or if `should_split_rhs` returns True. + return result.magic_trailing_comma is not None + + +class _BracketSplitComponent(Enum): + head = auto() + body = auto() + tail = auto() + + +def left_hand_split( + line: Line, _features: Collection[Feature], mode: Mode +) -> Iterator[Line]: + """Split line into many lines, starting with the first matching bracket pair. + + Note: this usually looks weird, only use this for function definitions. + Prefer RHS otherwise. This is why this function is not symmetrical with + :func:`right_hand_split` which also handles optional parentheses. + """ + for leaf_type in [token.LPAR, token.LSQB]: + tail_leaves: list[Leaf] = [] + body_leaves: list[Leaf] = [] + head_leaves: list[Leaf] = [] + current_leaves = head_leaves + matching_bracket: Optional[Leaf] = None + for leaf in line.leaves: + if ( + current_leaves is body_leaves + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket is matching_bracket + and isinstance(matching_bracket, Leaf) + ): + ensure_visible(leaf) + ensure_visible(matching_bracket) + current_leaves = tail_leaves if body_leaves else head_leaves + current_leaves.append(leaf) + if current_leaves is head_leaves: + if leaf.type == leaf_type: + matching_bracket = leaf + current_leaves = body_leaves + if matching_bracket and tail_leaves: + break + if not matching_bracket or not tail_leaves: + raise CannotSplit("No brackets found") + + head = bracket_split_build_line( + head_leaves, line, matching_bracket, component=_BracketSplitComponent.head + ) + body = bracket_split_build_line( + body_leaves, line, matching_bracket, component=_BracketSplitComponent.body + ) + tail = bracket_split_build_line( + tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail + ) + bracket_split_succeeded_or_raise(head, body, tail) + for result in (head, body, tail): + if result: + yield result + + +def right_hand_split( + line: Line, + mode: Mode, + features: Collection[Feature] = (), + omit: Collection[LeafID] = (), +) -> Iterator[Line]: + """Split line into many lines, starting with the last matching bracket pair. + + If the split was by optional parentheses, attempt splitting without them, too. + `omit` is a collection of closing bracket IDs that shouldn't be considered for + this split. + + Note: running this function modifies `bracket_depth` on the leaves of `line`. + """ + rhs_result = _first_right_hand_split(line, omit=omit) + yield from _maybe_split_omitting_optional_parens( + rhs_result, line, mode, features=features, omit=omit + ) + + +def _first_right_hand_split( + line: Line, + omit: Collection[LeafID] = (), +) -> RHSResult: + """Split the line into head, body, tail starting with the last bracket pair. + + Note: this function should not have side effects. It's relied upon by + _maybe_split_omitting_optional_parens to get an opinion whether to prefer + splitting on the right side of an assignment statement. + """ + tail_leaves: list[Leaf] = [] + body_leaves: list[Leaf] = [] + head_leaves: list[Leaf] = [] + current_leaves = tail_leaves + opening_bracket: Optional[Leaf] = None + closing_bracket: Optional[Leaf] = None + for leaf in reversed(line.leaves): + if current_leaves is body_leaves: + if leaf is opening_bracket: + current_leaves = head_leaves if body_leaves else tail_leaves + current_leaves.append(leaf) + if current_leaves is tail_leaves: + if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit: + opening_bracket = leaf.opening_bracket + closing_bracket = leaf + current_leaves = body_leaves + if not (opening_bracket and closing_bracket and head_leaves): + # If there is no opening or closing_bracket that means the split failed and + # all content is in the tail. Otherwise, if `head_leaves` are empty, it means + # the matching `opening_bracket` wasn't available on `line` anymore. + raise CannotSplit("No brackets found") + + tail_leaves.reverse() + body_leaves.reverse() + head_leaves.reverse() + + body: Optional[Line] = None + if ( + Preview.hug_parens_with_braces_and_square_brackets in line.mode + and tail_leaves[0].value + and tail_leaves[0].opening_bracket is head_leaves[-1] + ): + inner_body_leaves = list(body_leaves) + hugged_opening_leaves: list[Leaf] = [] + hugged_closing_leaves: list[Leaf] = [] + is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR] + unpacking_offset: int = 1 if is_unpacking else 0 + while ( + len(inner_body_leaves) >= 2 + unpacking_offset + and inner_body_leaves[-1].type in CLOSING_BRACKETS + and inner_body_leaves[-1].opening_bracket + is inner_body_leaves[unpacking_offset] + ): + if unpacking_offset: + hugged_opening_leaves.append(inner_body_leaves.pop(0)) + unpacking_offset = 0 + hugged_opening_leaves.append(inner_body_leaves.pop(0)) + hugged_closing_leaves.insert(0, inner_body_leaves.pop()) + + if hugged_opening_leaves and inner_body_leaves: + inner_body = bracket_split_build_line( + inner_body_leaves, + line, + hugged_opening_leaves[-1], + component=_BracketSplitComponent.body, + ) + if ( + line.mode.magic_trailing_comma + and inner_body_leaves[-1].type == token.COMMA + ): + should_hug = True + else: + line_length = line.mode.line_length - sum( + len(str(leaf)) + for leaf in hugged_opening_leaves + hugged_closing_leaves + ) + if is_line_short_enough( + inner_body, mode=replace(line.mode, line_length=line_length) + ): + # Do not hug if it fits on a single line. + should_hug = False + else: + should_hug = True + if should_hug: + body_leaves = inner_body_leaves + head_leaves.extend(hugged_opening_leaves) + tail_leaves = hugged_closing_leaves + tail_leaves + body = inner_body # No need to re-calculate the body again later. + + head = bracket_split_build_line( + head_leaves, line, opening_bracket, component=_BracketSplitComponent.head + ) + if body is None: + body = bracket_split_build_line( + body_leaves, line, opening_bracket, component=_BracketSplitComponent.body + ) + tail = bracket_split_build_line( + tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail + ) + bracket_split_succeeded_or_raise(head, body, tail) + return RHSResult(head, body, tail, opening_bracket, closing_bracket) + + +def _maybe_split_omitting_optional_parens( + rhs: RHSResult, + line: Line, + mode: Mode, + features: Collection[Feature] = (), + omit: Collection[LeafID] = (), +) -> Iterator[Line]: + if ( + Feature.FORCE_OPTIONAL_PARENTHESES not in features + # the opening bracket is an optional paren + and rhs.opening_bracket.type == token.LPAR + and not rhs.opening_bracket.value + # the closing bracket is an optional paren + and rhs.closing_bracket.type == token.RPAR + and not rhs.closing_bracket.value + # it's not an import (optional parens are the only thing we can split on + # in this case; attempting a split without them is a waste of time) + and not line.is_import + # and we can actually remove the parens + and can_omit_invisible_parens(rhs, mode.line_length) + ): + omit = {id(rhs.closing_bracket), *omit} + try: + # The RHSResult Omitting Optional Parens. + rhs_oop = _first_right_hand_split(line, omit=omit) + if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode): + yield from _maybe_split_omitting_optional_parens( + rhs_oop, line, mode, features=features, omit=omit + ) + return + + except CannotSplit as e: + # For chained assignments we want to use the previous successful split + if line.is_chained_assignment: + pass + + elif ( + not can_be_split(rhs.body) + and not is_line_short_enough(rhs.body, mode=mode) + and not ( + Preview.wrap_long_dict_values_in_parens + and rhs.opening_bracket.parent + and rhs.opening_bracket.parent.parent + and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker + ) + ): + raise CannotSplit( + "Splitting failed, body is still too long and can't be split." + ) from e + + elif ( + rhs.head.contains_multiline_strings() + or rhs.tail.contains_multiline_strings() + ): + raise CannotSplit( + "The current optional pair of parentheses is bound to fail to" + " satisfy the splitting algorithm because the head or the tail" + " contains multiline strings which by definition never fit one" + " line." + ) from e + + ensure_visible(rhs.opening_bracket) + ensure_visible(rhs.closing_bracket) + for result in (rhs.head, rhs.body, rhs.tail): + if result: + yield result + + +def _prefer_split_rhs_oop_over_rhs( + rhs_oop: RHSResult, rhs: RHSResult, mode: Mode +) -> bool: + """ + Returns whether we should prefer the result from a split omitting optional parens + (rhs_oop) over the original (rhs). + """ + # contains unsplittable type ignore + if ( + rhs_oop.head.contains_unsplittable_type_ignore() + or rhs_oop.body.contains_unsplittable_type_ignore() + or rhs_oop.tail.contains_unsplittable_type_ignore() + ): + return True + + # Retain optional parens around dictionary values + if ( + Preview.wrap_long_dict_values_in_parens + and rhs.opening_bracket.parent + and rhs.opening_bracket.parent.parent + and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker + and rhs.body.bracket_tracker.delimiters + ): + # Unless the split is inside the key + return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves) + + # the split is right after `=` + if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL): + return True + + # the left side of assignment contains brackets + if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]): + return True + + # the left side of assignment is short enough (the -1 is for the ending optional + # paren) + if not is_line_short_enough( + rhs.head, mode=replace(mode, line_length=mode.line_length - 1) + ): + return True + + # the left side of assignment won't explode further because of magic trailing comma + if rhs.head.magic_trailing_comma is not None: + return True + + # If we have multiple targets, we prefer more `=`s on the head vs pushing them to + # the body + rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL) + rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count( + token.EQUAL + ) + if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count: + return False + + has_closing_bracket_after_assign = False + for leaf in reversed(rhs_oop.head.leaves): + if leaf.type == token.EQUAL: + break + if leaf.type in CLOSING_BRACKETS: + has_closing_bracket_after_assign = True + break + return ( + # contains matching brackets after the `=` (done by checking there is a + # closing bracket) + has_closing_bracket_after_assign + or ( + # the split is actually from inside the optional parens (done by checking + # the first line still contains the `=`) + any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves) + # the first line is short enough + and is_line_short_enough(rhs_oop.head, mode=mode) + ) + ) + + +def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: + """Raise :exc:`CannotSplit` if the last left- or right-hand split failed. + + Do nothing otherwise. + + A left- or right-hand split is based on a pair of brackets. Content before + (and including) the opening bracket is left on one line, content inside the + brackets is put on a separate line, and finally content starting with and + following the closing bracket is put on a separate line. + + Those are called `head`, `body`, and `tail`, respectively. If the split + produced the same line (all content in `head`) or ended up with an empty `body` + and the `tail` is just the closing bracket, then it's considered failed. + """ + tail_len = len(str(tail).strip()) + if not body: + if tail_len == 0: + raise CannotSplit("Splitting brackets produced the same line") + + elif tail_len < 3: + raise CannotSplit( + f"Splitting brackets on an empty body to save {tail_len} characters is" + " not worth it" + ) + + +def _ensure_trailing_comma( + leaves: list[Leaf], original: Line, opening_bracket: Leaf +) -> bool: + if not leaves: + return False + # Ensure a trailing comma for imports + if original.is_import: + return True + # ...and standalone function arguments + if not original.is_def: + return False + if opening_bracket.value != "(": + return False + # Don't add commas if we already have any commas + if any( + leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves + ): + return False + + # Find a leaf with a parent (comments don't have parents) + leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None) + if leaf_with_parent is None: + return True + # Don't add commas inside parenthesized return annotations + if get_annotation_type(leaf_with_parent) == "return": + return False + # Don't add commas inside PEP 604 unions + if ( + leaf_with_parent.parent + and leaf_with_parent.parent.next_sibling + and leaf_with_parent.parent.next_sibling.type == token.VBAR + ): + return False + return True + + +def bracket_split_build_line( + leaves: list[Leaf], + original: Line, + opening_bracket: Leaf, + *, + component: _BracketSplitComponent, +) -> Line: + """Return a new line with given `leaves` and respective comments from `original`. + + If it's the head component, brackets will be tracked so trailing commas are + respected. + + If it's the body component, the result line is one-indented inside brackets and as + such has its first leaf's prefix normalized and a trailing comma added when + expected. + """ + result = Line(mode=original.mode, depth=original.depth) + if component is _BracketSplitComponent.body: + result.inside_brackets = True + result.depth += 1 + if _ensure_trailing_comma(leaves, original, opening_bracket): + for i in range(len(leaves) - 1, -1, -1): + if leaves[i].type == STANDALONE_COMMENT: + continue + + if leaves[i].type != token.COMMA: + new_comma = Leaf(token.COMMA, ",") + leaves.insert(i + 1, new_comma) + break + + leaves_to_track: set[LeafID] = set() + if component is _BracketSplitComponent.head: + leaves_to_track = get_leaves_inside_matching_brackets(leaves) + # Populate the line + for leaf in leaves: + result.append( + leaf, + preformatted=True, + track_bracket=id(leaf) in leaves_to_track, + ) + for comment_after in original.comments_after(leaf): + result.append(comment_after, preformatted=True) + if component is _BracketSplitComponent.body and should_split_line( + result, opening_bracket + ): + result.should_split_rhs = True + return result + + +def dont_increase_indentation(split_func: Transformer) -> Transformer: + """Normalize prefix of the first leaf in every line returned by `split_func`. + + This is a decorator over relevant split functions. + """ + + @wraps(split_func) + def split_wrapper( + line: Line, features: Collection[Feature], mode: Mode + ) -> Iterator[Line]: + for split_line in split_func(line, features, mode): + split_line.leaves[0].prefix = "" + yield split_line + + return split_wrapper + + +def _get_last_non_comment_leaf(line: Line) -> Optional[int]: + for leaf_idx in range(len(line.leaves) - 1, 0, -1): + if line.leaves[leaf_idx].type != STANDALONE_COMMENT: + return leaf_idx + return None + + +def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool: + if is_vararg(leaf, within={syms.typedargslist}): + return Feature.TRAILING_COMMA_IN_DEF in features + if is_vararg(leaf, within={syms.arglist, syms.argument}): + return Feature.TRAILING_COMMA_IN_CALL in features + return True + + +def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line: + if ( + safe + and delimiter_priority == COMMA_PRIORITY + and line.leaves[-1].type != token.COMMA + and line.leaves[-1].type != STANDALONE_COMMENT + ): + new_comma = Leaf(token.COMMA, ",") + line.append(new_comma) + return line + + +MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY} + + +@dont_increase_indentation +def delimiter_split( + line: Line, features: Collection[Feature], mode: Mode +) -> Iterator[Line]: + """Split according to delimiters of the highest priority. + + If the appropriate Features are given, the split will add trailing commas + also in function signatures and calls that contain `*` and `**`. + """ + if len(line.leaves) == 0: + raise CannotSplit("Line empty") from None + last_leaf = line.leaves[-1] + + bt = line.bracket_tracker + try: + delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)}) + except ValueError: + raise CannotSplit("No delimiters found") from None + + if ( + delimiter_priority == DOT_PRIORITY + and bt.delimiter_count_with_priority(delimiter_priority) == 1 + ): + raise CannotSplit("Splitting a single attribute from its owner looks wrong") + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + lowest_depth = sys.maxsize + trailing_comma_safe = True + + def append_to_line(leaf: Leaf) -> Iterator[Line]: + """Append `leaf` to current line or to new line if appending impossible.""" + nonlocal current_line + try: + current_line.append_safe(leaf, preformatted=True) + except ValueError: + yield current_line + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + current_line.append(leaf) + + def append_comments(leaf: Leaf) -> Iterator[Line]: + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + + last_non_comment_leaf = _get_last_non_comment_leaf(line) + for leaf_idx, leaf in enumerate(line.leaves): + yield from append_to_line(leaf) + + previous_priority = leaf_idx > 0 and bt.delimiters.get( + id(line.leaves[leaf_idx - 1]) + ) + if ( + previous_priority != delimiter_priority + or delimiter_priority in MIGRATE_COMMENT_DELIMITERS + ): + yield from append_comments(leaf) + + lowest_depth = min(lowest_depth, leaf.bracket_depth) + if trailing_comma_safe and leaf.bracket_depth == lowest_depth: + trailing_comma_safe = _can_add_trailing_comma(leaf, features) + + if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf: + current_line = _safe_add_trailing_comma( + trailing_comma_safe, delimiter_priority, current_line + ) + + leaf_priority = bt.delimiters.get(id(leaf)) + if leaf_priority == delimiter_priority: + if ( + leaf_idx + 1 < len(line.leaves) + and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS + ): + yield from append_comments(line.leaves[leaf_idx + 1]) + + yield current_line + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + + if current_line: + current_line = _safe_add_trailing_comma( + trailing_comma_safe, delimiter_priority, current_line + ) + yield current_line + + +@dont_increase_indentation +def standalone_comment_split( + line: Line, features: Collection[Feature], mode: Mode +) -> Iterator[Line]: + """Split standalone comments from the rest of the line.""" + if not line.contains_standalone_comments(): + raise CannotSplit("Line does not have any standalone comments") + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + + def append_to_line(leaf: Leaf) -> Iterator[Line]: + """Append `leaf` to current line or to new line if appending impossible.""" + nonlocal current_line + try: + current_line.append_safe(leaf, preformatted=True) + except ValueError: + yield current_line + + current_line = Line( + line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + current_line.append(leaf) + + for leaf in line.leaves: + yield from append_to_line(leaf) + + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + + if current_line: + yield current_line + + +def normalize_invisible_parens( # noqa: C901 + node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature] +) -> None: + """Make existing optional parentheses invisible or create new ones. + + `parens_after` is a set of string leaf values immediately after which parens + should be put. + + Standardizes on visible parentheses for single-element tuples, and keeps + existing visible parentheses for other tuples and generator expressions. + """ + for pc in list_comments(node.prefix, is_endmarker=False): + if pc.value in FMT_OFF: + # This `node` has a prefix with `# fmt: off`, don't mess with parens. + return + + # The multiple context managers grammar has a different pattern, thus this is + # separate from the for-loop below. This possibly wraps them in invisible parens, + # and later will be removed in remove_with_parens when needed. + if node.type == syms.with_stmt: + _maybe_wrap_cms_in_parens(node, mode, features) + + check_lpar = False + for index, child in enumerate(list(node.children)): + # Fixes a bug where invisible parens are not properly stripped from + # assignment statements that contain type annotations. + if isinstance(child, Node) and child.type == syms.annassign: + normalize_invisible_parens( + child, parens_after=parens_after, mode=mode, features=features + ) + + # Fixes a bug where invisible parens are not properly wrapped around + # case blocks. + if isinstance(child, Node) and child.type == syms.case_block: + normalize_invisible_parens( + child, parens_after={"case"}, mode=mode, features=features + ) + + # Add parentheses around if guards in case blocks + if isinstance(child, Node) and child.type == syms.guard: + normalize_invisible_parens( + child, parens_after={"if"}, mode=mode, features=features + ) + + # Add parentheses around long tuple unpacking in assignments. + if ( + index == 0 + and isinstance(child, Node) + and child.type == syms.testlist_star_expr + ): + check_lpar = True + + if check_lpar: + if ( + child.type == syms.atom + and node.type == syms.for_stmt + and isinstance(child.prev_sibling, Leaf) + and child.prev_sibling.type == token.NAME + and child.prev_sibling.value == "for" + ): + if maybe_make_parens_invisible_in_atom( + child, + parent=node, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(node, child, visible=False) + elif isinstance(child, Node) and node.type == syms.with_stmt: + remove_with_parens(child, node) + elif child.type == syms.atom: + if maybe_make_parens_invisible_in_atom( + child, + parent=node, + ): + wrap_in_parentheses(node, child, visible=False) + elif is_one_tuple(child): + wrap_in_parentheses(node, child, visible=True) + elif node.type == syms.import_from: + _normalize_import_from(node, child, index) + break + elif ( + index == 1 + and child.type == token.STAR + and node.type == syms.except_clause + ): + # In except* (PEP 654), the star is actually part of + # of the keyword. So we need to skip the insertion of + # invisible parentheses to work more precisely. + continue + + elif ( + isinstance(child, Leaf) + and child.next_sibling is not None + and child.next_sibling.type == token.COLON + and child.value == "case" + ): + # A special patch for "case case:" scenario, the second occurrence + # of case will be not parsed as a Python keyword. + break + + elif not is_multiline_string(child): + wrap_in_parentheses(node, child, visible=False) + + comma_check = child.type == token.COMMA + + check_lpar = isinstance(child, Leaf) and ( + child.value in parens_after or comma_check + ) + + +def _normalize_import_from(parent: Node, child: LN, index: int) -> None: + # "import from" nodes store parentheses directly as part of + # the statement + if is_lpar_token(child): + assert is_rpar_token(parent.children[-1]) + # make parentheses invisible + child.value = "" + parent.children[-1].value = "" + elif child.type != token.STAR: + # insert invisible parentheses + parent.insert_child(index, Leaf(token.LPAR, "")) + parent.append_child(Leaf(token.RPAR, "")) + + +def remove_await_parens(node: Node) -> None: + if node.children[0].type == token.AWAIT and len(node.children) > 1: + if ( + node.children[1].type == syms.atom + and node.children[1].children[0].type == token.LPAR + ): + if maybe_make_parens_invisible_in_atom( + node.children[1], + parent=node, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(node, node.children[1], visible=False) + + # Since await is an expression we shouldn't remove + # brackets in cases where this would change + # the AST due to operator precedence. + # Therefore we only aim to remove brackets around + # power nodes that aren't also await expressions themselves. + # https://peps.python.org/pep-0492/#updated-operator-precedence-table + # N.B. We've still removed any redundant nested brackets though :) + opening_bracket = cast(Leaf, node.children[1].children[0]) + closing_bracket = cast(Leaf, node.children[1].children[-1]) + bracket_contents = node.children[1].children[1] + if isinstance(bracket_contents, Node) and ( + bracket_contents.type != syms.power + or bracket_contents.children[0].type == token.AWAIT + or any( + isinstance(child, Leaf) and child.type == token.DOUBLESTAR + for child in bracket_contents.children + ) + ): + ensure_visible(opening_bracket) + ensure_visible(closing_bracket) + + +def _maybe_wrap_cms_in_parens( + node: Node, mode: Mode, features: Collection[Feature] +) -> None: + """When enabled and safe, wrap the multiple context managers in invisible parens. + + It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS. + """ + if ( + Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features + or len(node.children) <= 2 + # If it's an atom, it's already wrapped in parens. + or node.children[1].type == syms.atom + ): + return + colon_index: Optional[int] = None + for i in range(2, len(node.children)): + if node.children[i].type == token.COLON: + colon_index = i + break + if colon_index is not None: + lpar = Leaf(token.LPAR, "") + rpar = Leaf(token.RPAR, "") + context_managers = node.children[1:colon_index] + for child in context_managers: + child.remove() + # After wrapping, the with_stmt will look like this: + # with_stmt + # NAME 'with' + # atom + # LPAR '' + # testlist_gexp + # ... <-- context_managers + # /testlist_gexp + # RPAR '' + # /atom + # COLON ':' + new_child = Node( + syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar] + ) + node.insert_child(1, new_child) + + +def remove_with_parens(node: Node, parent: Node) -> None: + """Recursively hide optional parens in `with` statements.""" + # Removing all unnecessary parentheses in with statements in one pass is a tad + # complex as different variations of bracketed statements result in pretty + # different parse trees: + # + # with (open("file")) as f: # this is an asexpr_test + # ... + # + # with (open("file") as f): # this is an atom containing an + # ... # asexpr_test + # + # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA, + # ... # asexpr_test + # + # with (open("file") as f, open("file") as f): # an atom containing a + # ... # testlist_gexp which then + # # contains multiple asexpr_test(s) + if node.type == syms.atom: + if maybe_make_parens_invisible_in_atom( + node, + parent=parent, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(parent, node, visible=False) + if isinstance(node.children[1], Node): + remove_with_parens(node.children[1], node) + elif node.type == syms.testlist_gexp: + for child in node.children: + if isinstance(child, Node): + remove_with_parens(child, node) + elif node.type == syms.asexpr_test and not any( + leaf.type == token.COLONEQUAL for leaf in node.leaves() + ): + if maybe_make_parens_invisible_in_atom( + node.children[0], + parent=node, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(node, node.children[0], visible=False) + + +def maybe_make_parens_invisible_in_atom( + node: LN, + parent: LN, + remove_brackets_around_comma: bool = False, +) -> bool: + """If it's safe, make the parens in the atom `node` invisible, recursively. + Additionally, remove repeated, adjacent invisible parens from the atom `node` + as they are redundant. + + Returns whether the node should itself be wrapped in invisible parentheses. + """ + if ( + node.type not in (syms.atom, syms.expr) + or is_empty_tuple(node) + or is_one_tuple(node) + or (is_yield(node) and parent.type != syms.expr_stmt) + or ( + # This condition tries to prevent removing non-optional brackets + # around a tuple, however, can be a bit overzealous so we provide + # and option to skip this check for `for` and `with` statements. + not remove_brackets_around_comma + and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY + ) + or is_tuple_containing_walrus(node) + or is_tuple_containing_star(node) + or is_generator(node) + ): + return False + + if is_walrus_assignment(node): + if parent.type in [ + syms.annassign, + syms.expr_stmt, + syms.assert_stmt, + syms.return_stmt, + syms.except_clause, + syms.funcdef, + syms.with_stmt, + syms.tname, + # these ones aren't useful to end users, but they do please fuzzers + syms.for_stmt, + syms.del_stmt, + syms.for_stmt, + ]: + return False + + first = node.children[0] + last = node.children[-1] + if is_lpar_token(first) and is_rpar_token(last): + middle = node.children[1] + # make parentheses invisible + if ( + # If the prefix of `middle` includes a type comment with + # ignore annotation, then we do not remove the parentheses + not is_type_ignore_comment_string(middle.prefix.strip()) + ): + first.value = "" + last.value = "" + maybe_make_parens_invisible_in_atom( + middle, + parent=parent, + remove_brackets_around_comma=remove_brackets_around_comma, + ) + + if is_atom_with_invisible_parens(middle): + # Strip the invisible parens from `middle` by replacing + # it with the child in-between the invisible parens + middle.replace(middle.children[1]) + + if middle.children[0].prefix.strip(): + # Preserve comments before first paren + middle.children[1].prefix = ( + middle.children[0].prefix + middle.children[1].prefix + ) + + if middle.children[-1].prefix.strip(): + # Preserve comments before last paren + last.prefix = middle.children[-1].prefix + last.prefix + + return False + + return True + + +def should_split_line(line: Line, opening_bracket: Leaf) -> bool: + """Should `line` be immediately split with `delimiter_split()` after RHS?""" + + if not (opening_bracket.parent and opening_bracket.value in "[{("): + return False + + # We're essentially checking if the body is delimited by commas and there's more + # than one of them (we're excluding the trailing comma and if the delimiter priority + # is still commas, that means there's more). + exclude = set() + trailing_comma = False + try: + last_leaf = line.leaves[-1] + if last_leaf.type == token.COMMA: + trailing_comma = True + exclude.add(id(last_leaf)) + max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude) + except (IndexError, ValueError): + return False + + return max_priority == COMMA_PRIORITY and ( + (line.mode.magic_trailing_comma and trailing_comma) + # always explode imports + or opening_bracket.parent.type in {syms.atom, syms.import_from} + ) + + +def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]: + """Generate sets of closing bracket IDs that should be omitted in a RHS. + + Brackets can be omitted if the entire trailer up to and including + a preceding closing bracket fits in one line. + + Yielded sets are cumulative (contain results of previous yields, too). First + set is empty, unless the line should explode, in which case bracket pairs until + the one that needs to explode are omitted. + """ + + omit: set[LeafID] = set() + if not line.magic_trailing_comma: + yield omit + + length = 4 * line.depth + opening_bracket: Optional[Leaf] = None + closing_bracket: Optional[Leaf] = None + inner_brackets: set[LeafID] = set() + for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True): + length += leaf_length + if length > line_length: + break + + has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix) + if leaf.type == STANDALONE_COMMENT or has_inline_comment: + break + + if opening_bracket: + if leaf is opening_bracket: + opening_bracket = None + elif leaf.type in CLOSING_BRACKETS: + prev = line.leaves[index - 1] if index > 0 else None + if ( + prev + and prev.type == token.COMMA + and leaf.opening_bracket is not None + and not is_one_sequence_between( + leaf.opening_bracket, leaf, line.leaves + ) + ): + # Never omit bracket pairs with trailing commas. + # We need to explode on those. + break + + inner_brackets.add(id(leaf)) + elif leaf.type in CLOSING_BRACKETS: + prev = line.leaves[index - 1] if index > 0 else None + if prev and prev.type in OPENING_BRACKETS: + # Empty brackets would fail a split so treat them as "inner" + # brackets (e.g. only add them to the `omit` set if another + # pair of brackets was good enough. + inner_brackets.add(id(leaf)) + continue + + if closing_bracket: + omit.add(id(closing_bracket)) + omit.update(inner_brackets) + inner_brackets.clear() + yield omit + + if ( + prev + and prev.type == token.COMMA + and leaf.opening_bracket is not None + and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves) + ): + # Never omit bracket pairs with trailing commas. + # We need to explode on those. + break + + if leaf.value: + opening_bracket = leaf.opening_bracket + closing_bracket = leaf + + +def run_transformer( + line: Line, + transform: Transformer, + mode: Mode, + features: Collection[Feature], + *, + line_str: str = "", +) -> list[Line]: + if not line_str: + line_str = line_to_string(line) + result: list[Line] = [] + for transformed_line in transform(line, features, mode): + if str(transformed_line).strip("\n") == line_str: + raise CannotTransform("Line transformer returned an unchanged result") + + result.extend(transform_line(transformed_line, mode=mode, features=features)) + + features_set = set(features) + if ( + Feature.FORCE_OPTIONAL_PARENTHESES in features_set + or transform.__class__.__name__ != "rhs" + or not line.bracket_tracker.invisible + or any(bracket.value for bracket in line.bracket_tracker.invisible) + or line.contains_multiline_strings() + or result[0].contains_uncollapsable_type_comments() + or result[0].contains_unsplittable_type_ignore() + or is_line_short_enough(result[0], mode=mode) + # If any leaves have no parents (which _can_ occur since + # `transform(line)` potentially destroys the line's underlying node + # structure), then we can't proceed. Doing so would cause the below + # call to `append_leaves()` to fail. + or any(leaf.parent is None for leaf in line.leaves) + ): + return result + + line_copy = line.clone() + append_leaves(line_copy, line, line.leaves) + features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES} + second_opinion = run_transformer( + line_copy, transform, mode, features_fop, line_str=line_str + ) + if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion): + result = second_opinion + return result diff --git a/contrib/python/black/black/lines.py b/contrib/python/black/black/lines.py new file mode 100644 index 00000000000..2a719def3c9 --- /dev/null +++ b/contrib/python/black/black/lines.py @@ -0,0 +1,1075 @@ +import itertools +import math +from collections.abc import Callable, Iterator, Sequence +from dataclasses import dataclass, field +from typing import Optional, TypeVar, Union, cast + +from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, BracketTracker +from black.mode import Mode, Preview +from black.nodes import ( + BRACKETS, + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + TEST_DESCENDANTS, + child_towards, + is_docstring, + is_import, + is_multiline_string, + is_one_sequence_between, + is_type_comment, + is_type_ignore_comment, + is_with_or_async_with_stmt, + make_simple_prefix, + replace_child, + syms, + whitespace, +) +from black.strings import str_width +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node + +# types +T = TypeVar("T") +Index = int +LeafID = int +LN = Union[Leaf, Node] + + +@dataclass +class Line: + """Holds leaves and comments. Can be printed with `str(line)`.""" + + mode: Mode = field(repr=False) + depth: int = 0 + leaves: list[Leaf] = field(default_factory=list) + # keys ordered like `leaves` + comments: dict[LeafID, list[Leaf]] = field(default_factory=dict) + bracket_tracker: BracketTracker = field(default_factory=BracketTracker) + inside_brackets: bool = False + should_split_rhs: bool = False + magic_trailing_comma: Optional[Leaf] = None + + def append( + self, leaf: Leaf, preformatted: bool = False, track_bracket: bool = False + ) -> None: + """Add a new `leaf` to the end of the line. + + Unless `preformatted` is True, the `leaf` will receive a new consistent + whitespace prefix and metadata applied by :class:`BracketTracker`. + Trailing commas are maybe removed, unpacked for loop variables are + demoted from being delimiters. + + Inline comments are put aside. + """ + has_value = ( + leaf.type in BRACKETS + # empty fstring-middles must not be truncated + or leaf.type == token.FSTRING_MIDDLE + or bool(leaf.value.strip()) + ) + if not has_value: + return + + if token.COLON == leaf.type and self.is_class_paren_empty: + del self.leaves[-2:] + if self.leaves and not preformatted: + # Note: at this point leaf.prefix should be empty except for + # imports, for which we only preserve newlines. + leaf.prefix += whitespace( + leaf, + complex_subscript=self.is_complex_subscript(leaf), + mode=self.mode, + ) + if self.inside_brackets or not preformatted or track_bracket: + self.bracket_tracker.mark(leaf) + if self.mode.magic_trailing_comma: + if self.has_magic_trailing_comma(leaf): + self.magic_trailing_comma = leaf + elif self.has_magic_trailing_comma(leaf): + self.remove_trailing_comma() + if not self.append_comment(leaf): + self.leaves.append(leaf) + + def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: + """Like :func:`append()` but disallow invalid standalone comment structure. + + Raises ValueError when any `leaf` is appended after a standalone comment + or when a standalone comment is not the first leaf on the line. + """ + if ( + self.bracket_tracker.depth == 0 + or self.bracket_tracker.any_open_for_or_lambda() + ): + if self.is_comment: + raise ValueError("cannot append to standalone comments") + + if self.leaves and leaf.type == STANDALONE_COMMENT: + raise ValueError( + "cannot append standalone comments to a populated line" + ) + + self.append(leaf, preformatted=preformatted) + + @property + def is_comment(self) -> bool: + """Is this line a standalone comment?""" + return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT + + @property + def is_decorator(self) -> bool: + """Is this line a decorator?""" + return bool(self) and self.leaves[0].type == token.AT + + @property + def is_import(self) -> bool: + """Is this an import line?""" + return bool(self) and is_import(self.leaves[0]) + + @property + def is_with_or_async_with_stmt(self) -> bool: + """Is this a with_stmt line?""" + return bool(self) and is_with_or_async_with_stmt(self.leaves[0]) + + @property + def is_class(self) -> bool: + """Is this line a class definition?""" + return ( + bool(self) + and self.leaves[0].type == token.NAME + and self.leaves[0].value == "class" + ) + + @property + def is_stub_class(self) -> bool: + """Is this line a class definition with a body consisting only of "..."?""" + return self.is_class and self.leaves[-3:] == [ + Leaf(token.DOT, ".") for _ in range(3) + ] + + @property + def is_def(self) -> bool: + """Is this a function definition? (Also returns True for async defs.)""" + try: + first_leaf = self.leaves[0] + except IndexError: + return False + + try: + second_leaf: Optional[Leaf] = self.leaves[1] + except IndexError: + second_leaf = None + return (first_leaf.type == token.NAME and first_leaf.value == "def") or ( + first_leaf.type == token.ASYNC + and second_leaf is not None + and second_leaf.type == token.NAME + and second_leaf.value == "def" + ) + + @property + def is_stub_def(self) -> bool: + """Is this line a function definition with a body consisting only of "..."?""" + return self.is_def and self.leaves[-4:] == [Leaf(token.COLON, ":")] + [ + Leaf(token.DOT, ".") for _ in range(3) + ] + + @property + def is_class_paren_empty(self) -> bool: + """Is this a class with no base classes but using parentheses? + + Those are unnecessary and should be removed. + """ + return ( + bool(self) + and len(self.leaves) == 4 + and self.is_class + and self.leaves[2].type == token.LPAR + and self.leaves[2].value == "(" + and self.leaves[3].type == token.RPAR + and self.leaves[3].value == ")" + ) + + @property + def _is_triple_quoted_string(self) -> bool: + """Is the line a triple quoted string?""" + if not self or self.leaves[0].type != token.STRING: + return False + value = self.leaves[0].value + if value.startswith(('"""', "'''")): + return True + if value.startswith(("r'''", 'r"""', "R'''", 'R"""')): + return True + return False + + @property + def is_docstring(self) -> bool: + """Is the line a docstring?""" + return bool(self) and is_docstring(self.leaves[0]) + + @property + def is_chained_assignment(self) -> bool: + """Is the line a chained assignment""" + return [leaf.type for leaf in self.leaves].count(token.EQUAL) > 1 + + @property + def opens_block(self) -> bool: + """Does this line open a new level of indentation.""" + if len(self.leaves) == 0: + return False + return self.leaves[-1].type == token.COLON + + def is_fmt_pass_converted( + self, *, first_leaf_matches: Optional[Callable[[Leaf], bool]] = None + ) -> bool: + """Is this line converted from fmt off/skip code? + + If first_leaf_matches is not None, it only returns True if the first + leaf of converted code matches. + """ + if len(self.leaves) != 1: + return False + leaf = self.leaves[0] + if ( + leaf.type != STANDALONE_COMMENT + or leaf.fmt_pass_converted_first_leaf is None + ): + return False + return first_leaf_matches is None or first_leaf_matches( + leaf.fmt_pass_converted_first_leaf + ) + + def contains_standalone_comments(self) -> bool: + """If so, needs to be split before emitting.""" + for leaf in self.leaves: + if leaf.type == STANDALONE_COMMENT: + return True + + return False + + def contains_implicit_multiline_string_with_comments(self) -> bool: + """Chck if we have an implicit multiline string with comments on the line""" + for leaf_type, leaf_group_iterator in itertools.groupby( + self.leaves, lambda leaf: leaf.type + ): + if leaf_type != token.STRING: + continue + leaf_list = list(leaf_group_iterator) + if len(leaf_list) == 1: + continue + for leaf in leaf_list: + if self.comments_after(leaf): + return True + return False + + def contains_uncollapsable_type_comments(self) -> bool: + ignored_ids = set() + try: + last_leaf = self.leaves[-1] + ignored_ids.add(id(last_leaf)) + if last_leaf.type == token.COMMA or ( + last_leaf.type == token.RPAR and not last_leaf.value + ): + # When trailing commas or optional parens are inserted by Black for + # consistency, comments after the previous last element are not moved + # (they don't have to, rendering will still be correct). So we ignore + # trailing commas and invisible. + last_leaf = self.leaves[-2] + ignored_ids.add(id(last_leaf)) + except IndexError: + return False + + # A type comment is uncollapsable if it is attached to a leaf + # that isn't at the end of the line (since that could cause it + # to get associated to a different argument) or if there are + # comments before it (since that could cause it to get hidden + # behind a comment. + comment_seen = False + for leaf_id, comments in self.comments.items(): + for comment in comments: + if is_type_comment(comment): + if comment_seen or ( + not is_type_ignore_comment(comment) + and leaf_id not in ignored_ids + ): + return True + + comment_seen = True + + return False + + def contains_unsplittable_type_ignore(self) -> bool: + if not self.leaves: + return False + + # If a 'type: ignore' is attached to the end of a line, we + # can't split the line, because we can't know which of the + # subexpressions the ignore was meant to apply to. + # + # We only want this to apply to actual physical lines from the + # original source, though: we don't want the presence of a + # 'type: ignore' at the end of a multiline expression to + # justify pushing it all onto one line. Thus we + # (unfortunately) need to check the actual source lines and + # only report an unsplittable 'type: ignore' if this line was + # one line in the original code. + + # Grab the first and last line numbers, skipping generated leaves + first_line = next((leaf.lineno for leaf in self.leaves if leaf.lineno != 0), 0) + last_line = next( + (leaf.lineno for leaf in reversed(self.leaves) if leaf.lineno != 0), 0 + ) + + if first_line == last_line: + # We look at the last two leaves since a comma or an + # invisible paren could have been added at the end of the + # line. + for node in self.leaves[-2:]: + for comment in self.comments.get(id(node), []): + if is_type_ignore_comment(comment): + return True + + return False + + def contains_multiline_strings(self) -> bool: + return any(is_multiline_string(leaf) for leaf in self.leaves) + + def has_magic_trailing_comma(self, closing: Leaf) -> bool: + """Return True if we have a magic trailing comma, that is when: + - there's a trailing comma here + - it's not from single-element square bracket indexing + - it's not a one-tuple + """ + if not ( + closing.type in CLOSING_BRACKETS + and self.leaves + and self.leaves[-1].type == token.COMMA + ): + return False + + if closing.type == token.RBRACE: + return True + + if closing.type == token.RSQB: + if ( + closing.parent is not None + and closing.parent.type == syms.trailer + and closing.opening_bracket is not None + and is_one_sequence_between( + closing.opening_bracket, + closing, + self.leaves, + brackets=(token.LSQB, token.RSQB), + ) + ): + assert closing.prev_sibling is not None + assert closing.prev_sibling.type == syms.subscriptlist + return False + + return True + + if self.is_import: + return True + + if closing.opening_bracket is not None and not is_one_sequence_between( + closing.opening_bracket, closing, self.leaves + ): + return True + + return False + + def append_comment(self, comment: Leaf) -> bool: + """Add an inline or standalone comment to the line.""" + if ( + comment.type == STANDALONE_COMMENT + and self.bracket_tracker.any_open_brackets() + ): + comment.prefix = "" + return False + + if comment.type != token.COMMENT: + return False + + if not self.leaves: + comment.type = STANDALONE_COMMENT + comment.prefix = "" + return False + + last_leaf = self.leaves[-1] + if ( + last_leaf.type == token.RPAR + and not last_leaf.value + and last_leaf.parent + and len(list(last_leaf.parent.leaves())) <= 3 + and not is_type_comment(comment) + ): + # Comments on an optional parens wrapping a single leaf should belong to + # the wrapped node except if it's a type comment. Pinning the comment like + # this avoids unstable formatting caused by comment migration. + if len(self.leaves) < 2: + comment.type = STANDALONE_COMMENT + comment.prefix = "" + return False + + last_leaf = self.leaves[-2] + self.comments.setdefault(id(last_leaf), []).append(comment) + return True + + def comments_after(self, leaf: Leaf) -> list[Leaf]: + """Generate comments that should appear directly after `leaf`.""" + return self.comments.get(id(leaf), []) + + def remove_trailing_comma(self) -> None: + """Remove the trailing comma and moves the comments attached to it.""" + trailing_comma = self.leaves.pop() + trailing_comma_comments = self.comments.pop(id(trailing_comma), []) + self.comments.setdefault(id(self.leaves[-1]), []).extend( + trailing_comma_comments + ) + + def is_complex_subscript(self, leaf: Leaf) -> bool: + """Return True iff `leaf` is part of a slice with non-trivial exprs.""" + open_lsqb = self.bracket_tracker.get_open_lsqb() + if open_lsqb is None: + return False + + subscript_start = open_lsqb.next_sibling + + if isinstance(subscript_start, Node): + if subscript_start.type == syms.listmaker: + return False + + if subscript_start.type == syms.subscriptlist: + subscript_start = child_towards(subscript_start, leaf) + + return subscript_start is not None and any( + n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() + ) + + def enumerate_with_length( + self, is_reversed: bool = False + ) -> Iterator[tuple[Index, Leaf, int]]: + """Return an enumeration of leaves with their length. + + Stops prematurely on multiline strings and standalone comments. + """ + op = cast( + Callable[[Sequence[Leaf]], Iterator[tuple[Index, Leaf]]], + enumerate_reversed if is_reversed else enumerate, + ) + for index, leaf in op(self.leaves): + length = len(leaf.prefix) + len(leaf.value) + if "\n" in leaf.value: + return # Multiline strings, we can't continue. + + for comment in self.comments_after(leaf): + length += len(comment.value) + + yield index, leaf, length + + def clone(self) -> "Line": + return Line( + mode=self.mode, + depth=self.depth, + inside_brackets=self.inside_brackets, + should_split_rhs=self.should_split_rhs, + magic_trailing_comma=self.magic_trailing_comma, + ) + + def __str__(self) -> str: + """Render the line.""" + if not self: + return "\n" + + indent = " " * self.depth + leaves = iter(self.leaves) + first = next(leaves) + res = f"{first.prefix}{indent}{first.value}" + for leaf in leaves: + res += str(leaf) + for comment in itertools.chain.from_iterable(self.comments.values()): + res += str(comment) + + return res + "\n" + + def __bool__(self) -> bool: + """Return True if the line has leaves or comments.""" + return bool(self.leaves or self.comments) + + +@dataclass +class RHSResult: + """Intermediate split result from a right hand split.""" + + head: Line + body: Line + tail: Line + opening_bracket: Leaf + closing_bracket: Leaf + + +@dataclass +class LinesBlock: + """Class that holds information about a block of formatted lines. + + This is introduced so that the EmptyLineTracker can look behind the standalone + comments and adjust their empty lines for class or def lines. + """ + + mode: Mode + previous_block: Optional["LinesBlock"] + original_line: Line + before: int = 0 + content_lines: list[str] = field(default_factory=list) + after: int = 0 + form_feed: bool = False + + def all_lines(self) -> list[str]: + empty_line = str(Line(mode=self.mode)) + prefix = make_simple_prefix(self.before, self.form_feed, empty_line) + return [prefix] + self.content_lines + [empty_line * self.after] + + +@dataclass +class EmptyLineTracker: + """Provides a stateful method that returns the number of potential extra + empty lines needed before and after the currently processed line. + + Note: this tracker works on lines that haven't been split yet. It assumes + the prefix of the first leaf consists of optional newlines. Those newlines + are consumed by `maybe_empty_lines()` and included in the computation. + """ + + mode: Mode + previous_line: Optional[Line] = None + previous_block: Optional[LinesBlock] = None + previous_defs: list[Line] = field(default_factory=list) + semantic_leading_comment: Optional[LinesBlock] = None + + def maybe_empty_lines(self, current_line: Line) -> LinesBlock: + """Return the number of extra empty lines before and after the `current_line`. + + This is for separating `def`, `async def` and `class` with extra empty + lines (two on module-level). + """ + form_feed = ( + current_line.depth == 0 + and bool(current_line.leaves) + and "\f\n" in current_line.leaves[0].prefix + ) + before, after = self._maybe_empty_lines(current_line) + previous_after = self.previous_block.after if self.previous_block else 0 + before = max(0, before - previous_after) + if ( + # Always have one empty line after a module docstring + self.previous_block + and self.previous_block.previous_block is None + and len(self.previous_block.original_line.leaves) == 1 + and self.previous_block.original_line.is_docstring + and not (current_line.is_class or current_line.is_def) + ): + before = 1 + + block = LinesBlock( + mode=self.mode, + previous_block=self.previous_block, + original_line=current_line, + before=before, + after=after, + form_feed=form_feed, + ) + + # Maintain the semantic_leading_comment state. + if current_line.is_comment: + if self.previous_line is None or ( + not self.previous_line.is_decorator + # `or before` means this comment already has an empty line before + and (not self.previous_line.is_comment or before) + and (self.semantic_leading_comment is None or before) + ): + self.semantic_leading_comment = block + # `or before` means this decorator already has an empty line before + elif not current_line.is_decorator or before: + self.semantic_leading_comment = None + + self.previous_line = current_line + self.previous_block = block + return block + + def _maybe_empty_lines(self, current_line: Line) -> tuple[int, int]: # noqa: C901 + max_allowed = 1 + if current_line.depth == 0: + max_allowed = 1 if self.mode.is_pyi else 2 + + if current_line.leaves: + # Consume the first leaf's extra newlines. + first_leaf = current_line.leaves[0] + before = first_leaf.prefix.count("\n") + before = min(before, max_allowed) + first_leaf.prefix = "" + else: + before = 0 + + user_had_newline = bool(before) + depth = current_line.depth + + # Mutate self.previous_defs, remainder of this function should be pure + previous_def = None + while self.previous_defs and self.previous_defs[-1].depth >= depth: + previous_def = self.previous_defs.pop() + if current_line.is_def or current_line.is_class: + self.previous_defs.append(current_line) + + if self.previous_line is None: + # Don't insert empty lines before the first line in the file. + return 0, 0 + + if current_line.is_docstring: + if self.previous_line.is_class: + return 0, 1 + if self.previous_line.opens_block and self.previous_line.is_def: + return 0, 0 + + if previous_def is not None: + assert self.previous_line is not None + if self.mode.is_pyi: + if previous_def.is_class and not previous_def.is_stub_class: + before = 1 + elif depth and not current_line.is_def and self.previous_line.is_def: + # Empty lines between attributes and methods should be preserved. + before = 1 if user_had_newline else 0 + elif depth: + before = 0 + else: + before = 1 + else: + if depth: + before = 1 + elif ( + not depth + and previous_def.depth + and current_line.leaves[-1].type == token.COLON + and ( + current_line.leaves[0].value + not in ("with", "try", "for", "while", "if", "match") + ) + ): + # We shouldn't add two newlines between an indented function and + # a dependent non-indented clause. This is to avoid issues with + # conditional function definitions that are technically top-level + # and therefore get two trailing newlines, but look weird and + # inconsistent when they're followed by elif, else, etc. This is + # worse because these functions only get *one* preceding newline + # already. + before = 1 + else: + before = 2 + + if current_line.is_decorator or current_line.is_def or current_line.is_class: + return self._maybe_empty_lines_for_class_or_def( + current_line, before, user_had_newline + ) + + if ( + self.previous_line.is_import + and self.previous_line.depth == 0 + and current_line.depth == 0 + and not current_line.is_import + and Preview.always_one_newline_after_import in self.mode + ): + return 1, 0 + + if ( + self.previous_line.is_import + and not current_line.is_import + and not current_line.is_fmt_pass_converted(first_leaf_matches=is_import) + and depth == self.previous_line.depth + ): + return (before or 1), 0 + + return before, 0 + + def _maybe_empty_lines_for_class_or_def( # noqa: C901 + self, current_line: Line, before: int, user_had_newline: bool + ) -> tuple[int, int]: + assert self.previous_line is not None + + if self.previous_line.is_decorator: + if self.mode.is_pyi and current_line.is_stub_class: + # Insert an empty line after a decorated stub class + return 0, 1 + return 0, 0 + + if self.previous_line.depth < current_line.depth and ( + self.previous_line.is_class or self.previous_line.is_def + ): + if self.mode.is_pyi: + return 0, 0 + return 1 if user_had_newline else 0, 0 + + comment_to_add_newlines: Optional[LinesBlock] = None + if ( + self.previous_line.is_comment + and self.previous_line.depth == current_line.depth + and before == 0 + ): + slc = self.semantic_leading_comment + if ( + slc is not None + and slc.previous_block is not None + and not slc.previous_block.original_line.is_class + and not slc.previous_block.original_line.opens_block + and slc.before <= 1 + ): + comment_to_add_newlines = slc + else: + return 0, 0 + + if self.mode.is_pyi: + if current_line.is_class or self.previous_line.is_class: + if self.previous_line.depth < current_line.depth: + newlines = 0 + elif self.previous_line.depth > current_line.depth: + newlines = 1 + elif current_line.is_stub_class and self.previous_line.is_stub_class: + # No blank line between classes with an empty body + newlines = 0 + else: + newlines = 1 + # Don't inspect the previous line if it's part of the body of the previous + # statement in the same level, we always want a blank line if there's + # something with a body preceding. + elif self.previous_line.depth > current_line.depth: + newlines = 1 + elif ( + current_line.is_def or current_line.is_decorator + ) and not self.previous_line.is_def: + if current_line.depth: + # In classes empty lines between attributes and methods should + # be preserved. + newlines = min(1, before) + else: + # Blank line between a block of functions (maybe with preceding + # decorators) and a block of non-functions + newlines = 1 + else: + newlines = 0 + else: + newlines = 1 if current_line.depth else 2 + # If a user has left no space after a dummy implementation, don't insert + # new lines. This is useful for instance for @overload or Protocols. + if self.previous_line.is_stub_def and not user_had_newline: + newlines = 0 + if comment_to_add_newlines is not None: + previous_block = comment_to_add_newlines.previous_block + if previous_block is not None: + comment_to_add_newlines.before = ( + max(comment_to_add_newlines.before, newlines) - previous_block.after + ) + newlines = 0 + return newlines, 0 + + +def enumerate_reversed(sequence: Sequence[T]) -> Iterator[tuple[Index, T]]: + """Like `reversed(enumerate(sequence))` if that were possible.""" + index = len(sequence) - 1 + for element in reversed(sequence): + yield (index, element) + index -= 1 + + +def append_leaves( + new_line: Line, old_line: Line, leaves: list[Leaf], preformatted: bool = False +) -> None: + """ + Append leaves (taken from @old_line) to @new_line, making sure to fix the + underlying Node structure where appropriate. + + All of the leaves in @leaves are duplicated. The duplicates are then + appended to @new_line and used to replace their originals in the underlying + Node structure. Any comments attached to the old leaves are reattached to + the new leaves. + + Pre-conditions: + set(@leaves) is a subset of set(@old_line.leaves). + """ + for old_leaf in leaves: + new_leaf = Leaf(old_leaf.type, old_leaf.value) + replace_child(old_leaf, new_leaf) + new_line.append(new_leaf, preformatted=preformatted) + + for comment_leaf in old_line.comments_after(old_leaf): + new_line.append(comment_leaf, preformatted=True) + + +def is_line_short_enough( # noqa: C901 + line: Line, *, mode: Mode, line_str: str = "" +) -> bool: + """For non-multiline strings, return True if `line` is no longer than `line_length`. + For multiline strings, looks at the context around `line` to determine + if it should be inlined or split up. + Uses the provided `line_str` rendering, if any, otherwise computes a new one. + """ + if not line_str: + line_str = line_to_string(line) + + if Preview.multiline_string_handling not in mode: + return ( + str_width(line_str) <= mode.line_length + and "\n" not in line_str # multiline strings + and not line.contains_standalone_comments() + ) + + if line.contains_standalone_comments(): + return False + if "\n" not in line_str: + # No multiline strings (MLS) present + return str_width(line_str) <= mode.line_length + + first, *_, last = line_str.split("\n") + if str_width(first) > mode.line_length or str_width(last) > mode.line_length: + return False + + # Traverse the AST to examine the context of the multiline string (MLS), + # tracking aspects such as depth and comma existence, + # to determine whether to split the MLS or keep it together. + # Depth (which is based on the existing bracket_depth concept) + # is needed to determine nesting level of the MLS. + # Includes special case for trailing commas. + commas: list[int] = [] # tracks number of commas per depth level + multiline_string: Optional[Leaf] = None + # store the leaves that contain parts of the MLS + multiline_string_contexts: list[LN] = [] + + max_level_to_update: Union[int, float] = math.inf # track the depth of the MLS + for i, leaf in enumerate(line.leaves): + if max_level_to_update == math.inf: + had_comma: Optional[int] = None + if leaf.bracket_depth + 1 > len(commas): + commas.append(0) + elif leaf.bracket_depth + 1 < len(commas): + had_comma = commas.pop() + if ( + had_comma is not None + and multiline_string is not None + and multiline_string.bracket_depth == leaf.bracket_depth + 1 + ): + # Have left the level with the MLS, stop tracking commas + max_level_to_update = leaf.bracket_depth + if had_comma > 0: + # MLS was in parens with at least one comma - force split + return False + + if leaf.bracket_depth <= max_level_to_update and leaf.type == token.COMMA: + # Inside brackets, ignore trailing comma + # directly after MLS/MLS-containing expression + ignore_ctxs: list[Optional[LN]] = [None] + ignore_ctxs += multiline_string_contexts + if (line.inside_brackets or leaf.bracket_depth > 0) and ( + i != len(line.leaves) - 1 or leaf.prev_sibling not in ignore_ctxs + ): + commas[leaf.bracket_depth] += 1 + if max_level_to_update != math.inf: + max_level_to_update = min(max_level_to_update, leaf.bracket_depth) + + if is_multiline_string(leaf): + if len(multiline_string_contexts) > 0: + # >1 multiline string cannot fit on a single line - force split + return False + multiline_string = leaf + ctx: LN = leaf + # fetch the leaf components of the MLS in the AST + while str(ctx) in line_str: + multiline_string_contexts.append(ctx) + if ctx.parent is None: + break + ctx = ctx.parent + + # May not have a triple-quoted multiline string at all, + # in case of a regular string with embedded newlines and line continuations + if len(multiline_string_contexts) == 0: + return True + + return all(val == 0 for val in commas) + + +def can_be_split(line: Line) -> bool: + """Return False if the line cannot be split *for sure*. + + This is not an exhaustive search but a cheap heuristic that we can use to + avoid some unfortunate formattings (mostly around wrapping unsplittable code + in unnecessary parentheses). + """ + leaves = line.leaves + if len(leaves) < 2: + return False + + if leaves[0].type == token.STRING and leaves[1].type == token.DOT: + call_count = 0 + dot_count = 0 + next = leaves[-1] + for leaf in leaves[-2::-1]: + if leaf.type in OPENING_BRACKETS: + if next.type not in CLOSING_BRACKETS: + return False + + call_count += 1 + elif leaf.type == token.DOT: + dot_count += 1 + elif leaf.type == token.NAME: + if not (next.type == token.DOT or next.type in OPENING_BRACKETS): + return False + + elif leaf.type not in CLOSING_BRACKETS: + return False + + if dot_count > 1 and call_count > 1: + return False + + return True + + +def can_omit_invisible_parens( + rhs: RHSResult, + line_length: int, +) -> bool: + """Does `rhs.body` have a shape safe to reformat without optional parens around it? + + Returns True for only a subset of potentially nice looking formattings but + the point is to not return false positives that end up producing lines that + are too long. + """ + line = rhs.body + + # We need optional parens in order to split standalone comments to their own lines + # if there are no nested parens around the standalone comments + closing_bracket: Optional[Leaf] = None + for leaf in reversed(line.leaves): + if closing_bracket and leaf is closing_bracket.opening_bracket: + closing_bracket = None + if leaf.type == STANDALONE_COMMENT and not closing_bracket: + return False + if ( + not closing_bracket + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket in line.leaves + and leaf.value + ): + closing_bracket = leaf + + bt = line.bracket_tracker + if not bt.delimiters: + # Without delimiters the optional parentheses are useless. + return True + + max_priority = bt.max_delimiter_priority() + delimiter_count = bt.delimiter_count_with_priority(max_priority) + if delimiter_count > 1: + # With more than one delimiter of a kind the optional parentheses read better. + return False + + if delimiter_count == 1: + if max_priority == COMMA_PRIORITY and rhs.head.is_with_or_async_with_stmt: + # For two context manager with statements, the optional parentheses read + # better. In this case, `rhs.body` is the context managers part of + # the with statement. `rhs.head` is the `with (` part on the previous + # line. + return False + # Otherwise it may also read better, but we don't do it today and requires + # careful considerations for all possible cases. See + # https://github.com/psf/black/issues/2156. + + if max_priority == DOT_PRIORITY: + # A single stranded method call doesn't require optional parentheses. + return True + + assert len(line.leaves) >= 2, "Stranded delimiter" + + # With a single delimiter, omit if the expression starts or ends with + # a bracket. + first = line.leaves[0] + second = line.leaves[1] + if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS: + if _can_omit_opening_paren(line, first=first, line_length=line_length): + return True + + # Note: we are not returning False here because a line might have *both* + # a leading opening bracket and a trailing closing bracket. If the + # opening bracket doesn't match our rule, maybe the closing will. + + penultimate = line.leaves[-2] + last = line.leaves[-1] + + if ( + last.type == token.RPAR + or last.type == token.RBRACE + or ( + # don't use indexing for omitting optional parentheses; + # it looks weird + last.type == token.RSQB + and last.parent + and last.parent.type != syms.trailer + ) + ): + if penultimate.type in OPENING_BRACKETS: + # Empty brackets don't help. + return False + + if is_multiline_string(first): + # Additional wrapping of a multiline string in this situation is + # unnecessary. + return True + + if _can_omit_closing_paren(line, last=last, line_length=line_length): + return True + + return False + + +def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: + """See `can_omit_invisible_parens`.""" + remainder = False + length = 4 * line.depth + _index = -1 + for _index, leaf, leaf_length in line.enumerate_with_length(): + if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: + remainder = True + if remainder: + length += leaf_length + if length > line_length: + break + + if leaf.type in OPENING_BRACKETS: + # There are brackets we can further split on. + remainder = False + + else: + # checked the entire string and line length wasn't exceeded + if len(line.leaves) == _index + 1: + return True + + return False + + +def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: + """See `can_omit_invisible_parens`.""" + length = 4 * line.depth + seen_other_brackets = False + for _index, leaf, leaf_length in line.enumerate_with_length(): + length += leaf_length + if leaf is last.opening_bracket: + if seen_other_brackets or length <= line_length: + return True + + elif leaf.type in OPENING_BRACKETS: + # There are brackets we can further split on. + seen_other_brackets = True + + return False + + +def line_to_string(line: Line) -> str: + """Returns the string representation of @line. + + WARNING: This is known to be computationally expensive. + """ + return str(line).strip("\n") diff --git a/contrib/python/black/black/mode.py b/contrib/python/black/black/mode.py new file mode 100644 index 00000000000..7335bd12078 --- /dev/null +++ b/contrib/python/black/black/mode.py @@ -0,0 +1,286 @@ +"""Data structures configuring Black behavior. + +Mostly around Python language feature support per version and Black configuration +chosen by the user. +""" + +from dataclasses import dataclass, field +from enum import Enum, auto +from hashlib import sha256 +from operator import attrgetter +from typing import Final + +from black.const import DEFAULT_LINE_LENGTH + + +class TargetVersion(Enum): + PY33 = 3 + PY34 = 4 + PY35 = 5 + PY36 = 6 + PY37 = 7 + PY38 = 8 + PY39 = 9 + PY310 = 10 + PY311 = 11 + PY312 = 12 + PY313 = 13 + + def pretty(self) -> str: + assert self.name[:2] == "PY" + return f"Python {self.name[2]}.{self.name[3:]}" + + +class Feature(Enum): + F_STRINGS = 2 + NUMERIC_UNDERSCORES = 3 + TRAILING_COMMA_IN_CALL = 4 + TRAILING_COMMA_IN_DEF = 5 + # The following two feature-flags are mutually exclusive, and exactly one should be + # set for every version of python. + ASYNC_IDENTIFIERS = 6 + ASYNC_KEYWORDS = 7 + ASSIGNMENT_EXPRESSIONS = 8 + POS_ONLY_ARGUMENTS = 9 + RELAXED_DECORATORS = 10 + PATTERN_MATCHING = 11 + UNPACKING_ON_FLOW = 12 + ANN_ASSIGN_EXTENDED_RHS = 13 + EXCEPT_STAR = 14 + VARIADIC_GENERICS = 15 + DEBUG_F_STRINGS = 16 + PARENTHESIZED_CONTEXT_MANAGERS = 17 + TYPE_PARAMS = 18 + FSTRING_PARSING = 19 + TYPE_PARAM_DEFAULTS = 20 + FORCE_OPTIONAL_PARENTHESES = 50 + + # __future__ flags + FUTURE_ANNOTATIONS = 51 + + +FUTURE_FLAG_TO_FEATURE: Final = { + "annotations": Feature.FUTURE_ANNOTATIONS, +} + + +VERSION_TO_FEATURES: dict[TargetVersion, set[Feature]] = { + TargetVersion.PY33: {Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY34: {Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY35: {Feature.TRAILING_COMMA_IN_CALL, Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY36: { + Feature.F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_IDENTIFIERS, + }, + TargetVersion.PY37: { + Feature.F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + }, + TargetVersion.PY38: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + }, + TargetVersion.PY39: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + }, + TargetVersion.PY310: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + }, + TargetVersion.PY311: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + }, + TargetVersion.PY312: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + Feature.FSTRING_PARSING, + }, + TargetVersion.PY313: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + Feature.FSTRING_PARSING, + Feature.TYPE_PARAM_DEFAULTS, + }, +} + + +def supports_feature(target_versions: set[TargetVersion], feature: Feature) -> bool: + return all(feature in VERSION_TO_FEATURES[version] for version in target_versions) + + +class Preview(Enum): + """Individual preview style features.""" + + # NOTE: string_processing requires wrap_long_dict_values_in_parens + # for https://github.com/psf/black/issues/3117 to be fixed. + string_processing = auto() + hug_parens_with_braces_and_square_brackets = auto() + wrap_long_dict_values_in_parens = auto() + multiline_string_handling = auto() + always_one_newline_after_import = auto() + + +UNSTABLE_FEATURES: set[Preview] = { + # Many issues, see summary in https://github.com/psf/black/issues/4042 + Preview.string_processing, + # See issue #4159 + Preview.multiline_string_handling, + # See issue #4036 (crash), #4098, #4099 (proposed tweaks) + Preview.hug_parens_with_braces_and_square_brackets, +} + + +class Deprecated(UserWarning): + """Visible deprecation warning.""" + + +_MAX_CACHE_KEY_PART_LENGTH: Final = 32 + + +@dataclass +class Mode: + target_versions: set[TargetVersion] = field(default_factory=set) + line_length: int = DEFAULT_LINE_LENGTH + string_normalization: bool = True + is_pyi: bool = False + is_ipynb: bool = False + skip_source_first_line: bool = False + magic_trailing_comma: bool = True + python_cell_magics: set[str] = field(default_factory=set) + preview: bool = False + unstable: bool = False + enabled_features: set[Preview] = field(default_factory=set) + + def __contains__(self, feature: Preview) -> bool: + """ + Provide `Preview.FEATURE in Mode` syntax that mirrors the ``preview`` flag. + + In unstable mode, all features are enabled. In preview mode, all features + except those in UNSTABLE_FEATURES are enabled. Any features in + `self.enabled_features` are also enabled. + """ + if self.unstable: + return True + if feature in self.enabled_features: + return True + return self.preview and feature not in UNSTABLE_FEATURES + + def get_cache_key(self) -> str: + if self.target_versions: + version_str = ",".join( + str(version.value) + for version in sorted(self.target_versions, key=attrgetter("value")) + ) + else: + version_str = "-" + if len(version_str) > _MAX_CACHE_KEY_PART_LENGTH: + version_str = sha256(version_str.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] + features_and_magics = ( + ",".join(sorted(f.name for f in self.enabled_features)) + + "@" + + ",".join(sorted(self.python_cell_magics)) + ) + if len(features_and_magics) > _MAX_CACHE_KEY_PART_LENGTH: + features_and_magics = sha256(features_and_magics.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] + parts = [ + version_str, + str(self.line_length), + str(int(self.string_normalization)), + str(int(self.is_pyi)), + str(int(self.is_ipynb)), + str(int(self.skip_source_first_line)), + str(int(self.magic_trailing_comma)), + str(int(self.preview)), + str(int(self.unstable)), + features_and_magics, + ] + return ".".join(parts) diff --git a/contrib/python/black/black/nodes.py b/contrib/python/black/black/nodes.py new file mode 100644 index 00000000000..3b74e2db0be --- /dev/null +++ b/contrib/python/black/black/nodes.py @@ -0,0 +1,1049 @@ +""" +blib2to3 Node/Leaf transformation-related utility functions. +""" + +import sys +from collections.abc import Iterator +from typing import Final, Generic, Literal, Optional, TypeVar, Union + +if sys.version_info >= (3, 10): + from typing import TypeGuard +else: + from typing_extensions import TypeGuard + +from mypy_extensions import mypyc_attr + +from black.cache import CACHE_DIR +from black.mode import Mode +from black.strings import get_string_prefix, has_triple_quotes +from blib2to3 import pygram +from blib2to3.pgen2 import token +from blib2to3.pytree import NL, Leaf, Node, type_repr + +pygram.initialize(CACHE_DIR) +syms: Final = pygram.python_symbols + + +# types +T = TypeVar("T") +LN = Union[Leaf, Node] +LeafID = int +NodeType = int + + +WHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE} +STATEMENT: Final = { + syms.if_stmt, + syms.while_stmt, + syms.for_stmt, + syms.try_stmt, + syms.except_clause, + syms.with_stmt, + syms.funcdef, + syms.classdef, + syms.match_stmt, + syms.case_block, +} +STANDALONE_COMMENT: Final = 153 +token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT" +LOGIC_OPERATORS: Final = {"and", "or"} +COMPARATORS: Final = { + token.LESS, + token.GREATER, + token.EQEQUAL, + token.NOTEQUAL, + token.LESSEQUAL, + token.GREATEREQUAL, +} +MATH_OPERATORS: Final = { + token.VBAR, + token.CIRCUMFLEX, + token.AMPER, + token.LEFTSHIFT, + token.RIGHTSHIFT, + token.PLUS, + token.MINUS, + token.STAR, + token.SLASH, + token.DOUBLESLASH, + token.PERCENT, + token.AT, + token.TILDE, + token.DOUBLESTAR, +} +STARS: Final = {token.STAR, token.DOUBLESTAR} +VARARGS_SPECIALS: Final = STARS | {token.SLASH} +VARARGS_PARENTS: Final = { + syms.arglist, + syms.argument, # double star in arglist + syms.trailer, # single argument to call + syms.typedargslist, + syms.varargslist, # lambdas +} +UNPACKING_PARENTS: Final = { + syms.atom, # single element of a list or set literal + syms.dictsetmaker, + syms.listmaker, + syms.testlist_gexp, + syms.testlist_star_expr, + syms.subject_expr, + syms.pattern, +} +TEST_DESCENDANTS: Final = { + syms.test, + syms.lambdef, + syms.or_test, + syms.and_test, + syms.not_test, + syms.comparison, + syms.star_expr, + syms.expr, + syms.xor_expr, + syms.and_expr, + syms.shift_expr, + syms.arith_expr, + syms.trailer, + syms.term, + syms.power, + syms.namedexpr_test, +} +TYPED_NAMES: Final = {syms.tname, syms.tname_star} +ASSIGNMENTS: Final = { + "=", + "+=", + "-=", + "*=", + "@=", + "/=", + "%=", + "&=", + "|=", + "^=", + "<<=", + ">>=", + "**=", + "//=", + ":", +} + +IMPLICIT_TUPLE: Final = {syms.testlist, syms.testlist_star_expr, syms.exprlist} +BRACKET: Final = { + token.LPAR: token.RPAR, + token.LSQB: token.RSQB, + token.LBRACE: token.RBRACE, +} +OPENING_BRACKETS: Final = set(BRACKET.keys()) +CLOSING_BRACKETS: Final = set(BRACKET.values()) +BRACKETS: Final = OPENING_BRACKETS | CLOSING_BRACKETS +ALWAYS_NO_SPACE: Final = CLOSING_BRACKETS | { + token.COMMA, + STANDALONE_COMMENT, + token.FSTRING_MIDDLE, + token.FSTRING_END, + token.BANG, +} + +RARROW = 55 + + +@mypyc_attr(allow_interpreted_subclasses=True) +class Visitor(Generic[T]): + """Basic lib2to3 visitor that yields things of type `T` on `visit()`.""" + + def visit(self, node: LN) -> Iterator[T]: + """Main method to visit `node` and its children. + + It tries to find a `visit_*()` method for the given `node.type`, like + `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects. + If no dedicated `visit_*()` method is found, chooses `visit_default()` + instead. + + Then yields objects of type `T` from the selected visitor. + """ + if node.type < 256: + name = token.tok_name[node.type] + else: + name = str(type_repr(node.type)) + # We explicitly branch on whether a visitor exists (instead of + # using self.visit_default as the default arg to getattr) in order + # to save needing to create a bound method object and so mypyc can + # generate a native call to visit_default. + visitf = getattr(self, f"visit_{name}", None) + if visitf: + yield from visitf(node) + else: + yield from self.visit_default(node) + + def visit_default(self, node: LN) -> Iterator[T]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" + if isinstance(node, Node): + for child in node.children: + yield from self.visit(child) + + +def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # noqa: C901 + """Return whitespace prefix if needed for the given `leaf`. + + `complex_subscript` signals whether the given leaf is part of a subscription + which has non-trivial arguments, like arithmetic expressions or function calls. + """ + NO: Final[str] = "" + SPACE: Final[str] = " " + DOUBLESPACE: Final[str] = " " + t = leaf.type + p = leaf.parent + v = leaf.value + if t in ALWAYS_NO_SPACE: + return NO + + if t == token.COMMENT: + return DOUBLESPACE + + assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}" + if t == token.COLON and p.type not in { + syms.subscript, + syms.subscriptlist, + syms.sliceop, + }: + return NO + + if t == token.LBRACE and p.type == syms.fstring_replacement_field: + return NO + + prev = leaf.prev_sibling + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type in OPENING_BRACKETS: + return NO + + if t == token.COLON: + if prevp.type == token.COLON: + return NO + + elif prevp.type != token.COMMA and not complex_subscript: + return NO + + return SPACE + + if prevp.type == token.EQUAL: + if prevp.parent: + if prevp.parent.type in { + syms.arglist, + syms.argument, + syms.parameters, + syms.varargslist, + }: + return NO + + elif prevp.parent.type == syms.typedargslist: + # A bit hacky: if the equal sign has whitespace, it means we + # previously found it's a typed argument. So, we're using + # that, too. + return prevp.prefix + + elif ( + prevp.type == token.STAR + and parent_type(prevp) == syms.star_expr + and parent_type(prevp.parent) in (syms.subscriptlist, syms.tname_star) + ): + # No space between typevar tuples or unpacking them. + return NO + + elif prevp.type in VARARGS_SPECIALS: + if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS): + return NO + + elif prevp.type == token.COLON: + if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}: + return SPACE if complex_subscript else NO + + elif ( + prevp.parent + and prevp.parent.type == syms.factor + and prevp.type in MATH_OPERATORS + ): + return NO + + elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator: + # no space in decorators + return NO + + elif prev.type in OPENING_BRACKETS: + return NO + + elif prev.type == token.BANG: + return NO + + if p.type in {syms.parameters, syms.arglist}: + # untyped function signatures or calls + if not prev or prev.type != token.COMMA: + return NO + + elif p.type == syms.varargslist: + # lambdas + if prev and prev.type != token.COMMA: + return NO + + elif p.type == syms.typedargslist: + # typed function signatures + if not prev: + return NO + + if t == token.EQUAL: + if prev.type not in TYPED_NAMES: + return NO + + elif prev.type == token.EQUAL: + # A bit hacky: if the equal sign has whitespace, it means we + # previously found it's a typed argument. So, we're using that, too. + return prev.prefix + + elif prev.type != token.COMMA: + return NO + + elif p.type in TYPED_NAMES: + # type names + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type != token.COMMA: + return NO + + elif p.type == syms.trailer: + # attributes and calls + if t == token.LPAR or t == token.RPAR: + return NO + + if not prev: + if t == token.DOT or t == token.LSQB: + return NO + + elif prev.type != token.COMMA: + return NO + + elif p.type == syms.argument: + # single argument + if t == token.EQUAL: + return NO + + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type == token.LPAR: + return NO + + elif prev.type in {token.EQUAL} | VARARGS_SPECIALS: + return NO + + elif p.type == syms.decorator: + # decorators + return NO + + elif p.type == syms.dotted_name: + if prev: + return NO + + prevp = preceding_leaf(p) + if not prevp or prevp.type == token.AT or prevp.type == token.DOT: + return NO + + elif p.type == syms.classdef: + if t == token.LPAR: + return NO + + if prev and prev.type == token.LPAR: + return NO + + elif p.type in {syms.subscript, syms.sliceop}: + # indexing + if not prev: + assert p.parent is not None, "subscripts are always parented" + if p.parent.type == syms.subscriptlist: + return SPACE + + return NO + + elif t == token.COLONEQUAL or prev.type == token.COLONEQUAL: + return SPACE + + elif not complex_subscript: + return NO + + elif p.type == syms.atom: + if prev and t == token.DOT: + # dots, but not the first one. + return NO + + elif p.type == syms.dictsetmaker: + # dict unpacking + if prev and prev.type == token.DOUBLESTAR: + return NO + + elif p.type in {syms.factor, syms.star_expr}: + # unary ops + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type in OPENING_BRACKETS: + return NO + + prevp_parent = prevp.parent + assert prevp_parent is not None + if prevp.type == token.COLON and prevp_parent.type in { + syms.subscript, + syms.sliceop, + }: + return NO + + elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument: + return NO + + # TODO: add fstring here? + elif t in {token.NAME, token.NUMBER, token.STRING}: + return NO + + elif p.type == syms.import_from: + if t == token.DOT: + if prev and prev.type == token.DOT: + return NO + + elif t == token.NAME: + if v == "import": + return SPACE + + if prev and prev.type == token.DOT: + return NO + + elif p.type == syms.sliceop: + return NO + + elif p.type == syms.except_clause: + if t == token.STAR: + return NO + + return SPACE + + +def make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = "\n") -> str: + """Generate a normalized prefix string.""" + if form_feed: + return (empty_line * (nl_count - 1)) + "\f" + empty_line + return empty_line * nl_count + + +def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: + """Return the first leaf that precedes `node`, if any.""" + while node: + res = node.prev_sibling + if res: + if isinstance(res, Leaf): + return res + + try: + return list(res.leaves())[-1] + + except IndexError: + return None + + node = node.parent + return None + + +def prev_siblings_are(node: Optional[LN], tokens: list[Optional[NodeType]]) -> bool: + """Return if the `node` and its previous siblings match types against the provided + list of tokens; the provided `node`has its type matched against the last element in + the list. `None` can be used as the first element to declare that the start of the + list is anchored at the start of its parent's children.""" + if not tokens: + return True + if tokens[-1] is None: + return node is None + if not node: + return False + if node.type != tokens[-1]: + return False + return prev_siblings_are(node.prev_sibling, tokens[:-1]) + + +def parent_type(node: Optional[LN]) -> Optional[NodeType]: + """ + Returns: + @node.parent.type, if @node is not None and has a parent. + OR + None, otherwise. + """ + if node is None or node.parent is None: + return None + + return node.parent.type + + +def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]: + """Return the child of `ancestor` that contains `descendant`.""" + node: Optional[LN] = descendant + while node and node.parent != ancestor: + node = node.parent + return node + + +def replace_child(old_child: LN, new_child: LN) -> None: + """ + Side Effects: + * If @old_child.parent is set, replace @old_child with @new_child in + @old_child's underlying Node structure. + OR + * Otherwise, this function does nothing. + """ + parent = old_child.parent + if not parent: + return + + child_idx = old_child.remove() + if child_idx is not None: + parent.insert_child(child_idx, new_child) + + +def container_of(leaf: Leaf) -> LN: + """Return `leaf` or one of its ancestors that is the topmost container of it. + + By "container" we mean a node where `leaf` is the very first child. + """ + same_prefix = leaf.prefix + container: LN = leaf + while container: + parent = container.parent + if parent is None: + break + + if parent.children[0].prefix != same_prefix: + break + + if parent.type == syms.file_input: + break + + if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS: + break + + container = parent + return container + + +def first_leaf_of(node: LN) -> Optional[Leaf]: + """Returns the first leaf of the node tree.""" + if isinstance(node, Leaf): + return node + if node.children: + return first_leaf_of(node.children[0]) + else: + return None + + +def is_arith_like(node: LN) -> bool: + """Whether node is an arithmetic or a binary arithmetic expression""" + return node.type in { + syms.arith_expr, + syms.shift_expr, + syms.xor_expr, + syms.and_expr, + } + + +def is_docstring(node: NL) -> bool: + if isinstance(node, Leaf): + if node.type != token.STRING: + return False + + prefix = get_string_prefix(node.value) + if set(prefix).intersection("bBfF"): + return False + + if ( + node.parent + and node.parent.type == syms.simple_stmt + and not node.parent.prev_sibling + and node.parent.parent + and node.parent.parent.type == syms.file_input + ): + return True + + if prev_siblings_are( + node.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] + ): + return True + + # Multiline docstring on the same line as the `def`. + if prev_siblings_are(node.parent, [syms.parameters, token.COLON, syms.simple_stmt]): + # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python + # grammar. We're safe to return True without further checks. + return True + + return False + + +def is_empty_tuple(node: LN) -> bool: + """Return True if `node` holds an empty tuple.""" + return ( + node.type == syms.atom + and len(node.children) == 2 + and node.children[0].type == token.LPAR + and node.children[1].type == token.RPAR + ) + + +def is_one_tuple(node: LN) -> bool: + """Return True if `node` holds a tuple with one element, with or without parens.""" + if node.type == syms.atom: + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA + + return ( + node.type in IMPLICIT_TUPLE + and len(node.children) == 2 + and node.children[1].type == token.COMMA + ) + + +def is_tuple_containing_walrus(node: LN) -> bool: + """Return True if `node` holds a tuple that contains a walrus operator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.namedexpr_test for child in gexp.children) + + +def is_tuple_containing_star(node: LN) -> bool: + """Return True if `node` holds a tuple that contains a star operator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.star_expr for child in gexp.children) + + +def is_generator(node: LN) -> bool: + """Return True if `node` holds a generator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.old_comp_for for child in gexp.children) + + +def is_one_sequence_between( + opening: Leaf, + closing: Leaf, + leaves: list[Leaf], + brackets: tuple[int, int] = (token.LPAR, token.RPAR), +) -> bool: + """Return True if content between `opening` and `closing` is a one-sequence.""" + if (opening.type, closing.type) != brackets: + return False + + depth = closing.bracket_depth + 1 + for _opening_index, leaf in enumerate(leaves): + if leaf is opening: + break + + else: + raise LookupError("Opening paren not found in `leaves`") + + commas = 0 + _opening_index += 1 + for leaf in leaves[_opening_index:]: + if leaf is closing: + break + + bracket_depth = leaf.bracket_depth + if bracket_depth == depth and leaf.type == token.COMMA: + commas += 1 + if leaf.parent and leaf.parent.type in { + syms.arglist, + syms.typedargslist, + }: + commas += 1 + break + + return commas < 2 + + +def is_walrus_assignment(node: LN) -> bool: + """Return True iff `node` is of the shape ( test := test )""" + inner = unwrap_singleton_parenthesis(node) + return inner is not None and inner.type == syms.namedexpr_test + + +def is_simple_decorator_trailer(node: LN, last: bool = False) -> bool: + """Return True iff `node` is a trailer valid in a simple decorator""" + return node.type == syms.trailer and ( + ( + len(node.children) == 2 + and node.children[0].type == token.DOT + and node.children[1].type == token.NAME + ) + # last trailer can be an argument-less parentheses pair + or ( + last + and len(node.children) == 2 + and node.children[0].type == token.LPAR + and node.children[1].type == token.RPAR + ) + # last trailer can be arguments + or ( + last + and len(node.children) == 3 + and node.children[0].type == token.LPAR + # and node.children[1].type == syms.argument + and node.children[2].type == token.RPAR + ) + ) + + +def is_simple_decorator_expression(node: LN) -> bool: + """Return True iff `node` could be a 'dotted name' decorator + + This function takes the node of the 'namedexpr_test' of the new decorator + grammar and test if it would be valid under the old decorator grammar. + + The old grammar was: decorator: @ dotted_name [arguments] NEWLINE + The new grammar is : decorator: @ namedexpr_test NEWLINE + """ + if node.type == token.NAME: + return True + if node.type == syms.power: + if node.children: + return ( + node.children[0].type == token.NAME + and all(map(is_simple_decorator_trailer, node.children[1:-1])) + and ( + len(node.children) < 2 + or is_simple_decorator_trailer(node.children[-1], last=True) + ) + ) + return False + + +def is_yield(node: LN) -> bool: + """Return True if `node` holds a `yield` or `yield from` expression.""" + if node.type == syms.yield_expr: + return True + + if is_name_token(node) and node.value == "yield": + return True + + if node.type != syms.atom: + return False + + if len(node.children) != 3: + return False + + lpar, expr, rpar = node.children + if lpar.type == token.LPAR and rpar.type == token.RPAR: + return is_yield(expr) + + return False + + +def is_vararg(leaf: Leaf, within: set[NodeType]) -> bool: + """Return True if `leaf` is a star or double star in a vararg or kwarg. + + If `within` includes VARARGS_PARENTS, this applies to function signatures. + If `within` includes UNPACKING_PARENTS, it applies to right hand-side + extended iterable unpacking (PEP 3132) and additional unpacking + generalizations (PEP 448). + """ + if leaf.type not in VARARGS_SPECIALS or not leaf.parent: + return False + + p = leaf.parent + if p.type == syms.star_expr: + # Star expressions are also used as assignment targets in extended + # iterable unpacking (PEP 3132). See what its parent is instead. + if not p.parent: + return False + + p = p.parent + + return p.type in within + + +def is_fstring(node: Node) -> bool: + """Return True if the node is an f-string""" + return node.type == syms.fstring + + +def fstring_to_string(node: Node) -> Leaf: + """Converts an fstring node back to a string node.""" + string_without_prefix = str(node)[len(node.prefix) :] + string_leaf = Leaf(token.STRING, string_without_prefix, prefix=node.prefix) + string_leaf.lineno = node.get_lineno() or 0 + return string_leaf + + +def is_multiline_string(node: LN) -> bool: + """Return True if `leaf` is a multiline string that actually spans many lines.""" + if isinstance(node, Node) and is_fstring(node): + leaf = fstring_to_string(node) + elif isinstance(node, Leaf): + leaf = node + else: + return False + + return has_triple_quotes(leaf.value) and "\n" in leaf.value + + +def is_parent_function_or_class(node: Node) -> bool: + assert node.type in {syms.suite, syms.simple_stmt} + assert node.parent is not None + # Note this works for suites / simple_stmts in async def as well + return node.parent.type in {syms.funcdef, syms.classdef} + + +def is_function_or_class(node: Node) -> bool: + return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} + + +def is_stub_suite(node: Node) -> bool: + """Return True if `node` is a suite with a stub body.""" + if node.parent is not None and not is_parent_function_or_class(node): + return False + + # If there is a comment, we want to keep it. + if node.prefix.strip(): + return False + + if ( + len(node.children) != 4 + or node.children[0].type != token.NEWLINE + or node.children[1].type != token.INDENT + or node.children[3].type != token.DEDENT + ): + return False + + if node.children[3].prefix.strip(): + return False + + return is_stub_body(node.children[2]) + + +def is_stub_body(node: LN) -> bool: + """Return True if `node` is a simple statement containing an ellipsis.""" + if not isinstance(node, Node) or node.type != syms.simple_stmt: + return False + + if len(node.children) != 2: + return False + + child = node.children[0] + return ( + not child.prefix.strip() + and child.type == syms.atom + and len(child.children) == 3 + and all(leaf == Leaf(token.DOT, ".") for leaf in child.children) + ) + + +def is_atom_with_invisible_parens(node: LN) -> bool: + """Given a `LN`, determines whether it's an atom `node` with invisible + parens. Useful in dedupe-ing and normalizing parens. + """ + if isinstance(node, Leaf) or node.type != syms.atom: + return False + + first, last = node.children[0], node.children[-1] + return ( + isinstance(first, Leaf) + and first.type == token.LPAR + and first.value == "" + and isinstance(last, Leaf) + and last.type == token.RPAR + and last.value == "" + ) + + +def is_empty_par(leaf: Leaf) -> bool: + return is_empty_lpar(leaf) or is_empty_rpar(leaf) + + +def is_empty_lpar(leaf: Leaf) -> bool: + return leaf.type == token.LPAR and leaf.value == "" + + +def is_empty_rpar(leaf: Leaf) -> bool: + return leaf.type == token.RPAR and leaf.value == "" + + +def is_import(leaf: Leaf) -> bool: + """Return True if the given leaf starts an import statement.""" + p = leaf.parent + t = leaf.type + v = leaf.value + return bool( + t == token.NAME + and ( + (v == "import" and p and p.type == syms.import_name) + or (v == "from" and p and p.type == syms.import_from) + ) + ) + + +def is_with_or_async_with_stmt(leaf: Leaf) -> bool: + """Return True if the given leaf starts a with or async with statement.""" + return bool( + leaf.type == token.NAME + and leaf.value == "with" + and leaf.parent + and leaf.parent.type == syms.with_stmt + ) or bool( + leaf.type == token.ASYNC + and leaf.next_sibling + and leaf.next_sibling.type == syms.with_stmt + ) + + +def is_async_stmt_or_funcdef(leaf: Leaf) -> bool: + """Return True if the given leaf starts an async def/for/with statement. + + Note that `async def` can be either an `async_stmt` or `async_funcdef`, + the latter is used when it has decorators. + """ + return bool( + leaf.type == token.ASYNC + and leaf.parent + and leaf.parent.type in {syms.async_stmt, syms.async_funcdef} + ) + + +def is_type_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment. This function should only + be used for general type comments (excluding ignore annotations, which should + use `is_type_ignore_comment`). Note that general type comments are no longer + used in modern version of Python, this function may be deprecated in the future.""" + t = leaf.type + v = leaf.value + return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") + + +def is_type_ignore_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment with ignore annotation.""" + t = leaf.type + v = leaf.value + return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v) + + +def is_type_ignore_comment_string(value: str) -> bool: + """Return True if the given string match with type comment with + ignore annotation.""" + return value.startswith("# type: ignore") + + +def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: + """Wrap `child` in parentheses. + + This replaces `child` with an atom holding the parentheses and the old + child. That requires moving the prefix. + + If `visible` is False, the leaves will be valueless (and thus invisible). + """ + lpar = Leaf(token.LPAR, "(" if visible else "") + rpar = Leaf(token.RPAR, ")" if visible else "") + prefix = child.prefix + child.prefix = "" + index = child.remove() or 0 + new_child = Node(syms.atom, [lpar, child, rpar]) + new_child.prefix = prefix + parent.insert_child(index, new_child) + + +def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]: + """Returns `wrapped` if `node` is of the shape ( wrapped ). + + Parenthesis can be optional. Returns None otherwise""" + if len(node.children) != 3: + return None + + lpar, wrapped, rpar = node.children + if not (lpar.type == token.LPAR and rpar.type == token.RPAR): + return None + + return wrapped + + +def ensure_visible(leaf: Leaf) -> None: + """Make sure parentheses are visible. + + They could be invisible as part of some statements (see + :func:`normalize_invisible_parens` and :func:`visit_import_from`). + """ + if leaf.type == token.LPAR: + leaf.value = "(" + elif leaf.type == token.RPAR: + leaf.value = ")" + + +def is_name_token(nl: NL) -> TypeGuard[Leaf]: + return nl.type == token.NAME + + +def is_lpar_token(nl: NL) -> TypeGuard[Leaf]: + return nl.type == token.LPAR + + +def is_rpar_token(nl: NL) -> TypeGuard[Leaf]: + return nl.type == token.RPAR + + +def is_number_token(nl: NL) -> TypeGuard[Leaf]: + return nl.type == token.NUMBER + + +def get_annotation_type(leaf: Leaf) -> Literal["return", "param", None]: + """Returns the type of annotation this leaf is part of, if any.""" + ancestor = leaf.parent + while ancestor is not None: + if ancestor.prev_sibling and ancestor.prev_sibling.type == token.RARROW: + return "return" + if ancestor.parent and ancestor.parent.type == syms.tname: + return "param" + ancestor = ancestor.parent + return None + + +def is_part_of_annotation(leaf: Leaf) -> bool: + """Returns whether this leaf is part of a type annotation.""" + assert leaf.parent is not None + return get_annotation_type(leaf) is not None + + +def first_leaf(node: LN) -> Optional[Leaf]: + """Returns the first leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return first_leaf(node.children[0]) + + +def last_leaf(node: LN) -> Optional[Leaf]: + """Returns the last leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return last_leaf(node.children[-1]) + + +def furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN: + """Returns the furthest ancestor that has this leaf node as the last leaf.""" + node: LN = leaf + while node.parent and node.parent.children and node is node.parent.children[-1]: + node = node.parent + return node diff --git a/contrib/python/black/black/numerics.py b/contrib/python/black/black/numerics.py new file mode 100644 index 00000000000..3040de06fde --- /dev/null +++ b/contrib/python/black/black/numerics.py @@ -0,0 +1,61 @@ +""" +Formatting numeric literals. +""" + +from blib2to3.pytree import Leaf + + +def format_hex(text: str) -> str: + """ + Formats a hexadecimal string like "0x12B3" + """ + before, after = text[:2], text[2:] + return f"{before}{after.upper()}" + + +def format_scientific_notation(text: str) -> str: + """Formats a numeric string utilizing scientific notation""" + before, after = text.split("e") + sign = "" + if after.startswith("-"): + after = after[1:] + sign = "-" + elif after.startswith("+"): + after = after[1:] + before = format_float_or_int_string(before) + return f"{before}e{sign}{after}" + + +def format_complex_number(text: str) -> str: + """Formats a complex string like `10j`""" + number = text[:-1] + suffix = text[-1] + return f"{format_float_or_int_string(number)}{suffix}" + + +def format_float_or_int_string(text: str) -> str: + """Formats a float string like "1.0".""" + if "." not in text: + return text + + before, after = text.split(".") + return f"{before or 0}.{after or 0}" + + +def normalize_numeric_literal(leaf: Leaf) -> None: + """Normalizes numeric (float, int, and complex) literals. + + All letters used in the representation are normalized to lowercase.""" + text = leaf.value.lower() + if text.startswith(("0o", "0b")): + # Leave octal and binary literals alone. + pass + elif text.startswith("0x"): + text = format_hex(text) + elif "e" in text: + text = format_scientific_notation(text) + elif text.endswith("j"): + text = format_complex_number(text) + else: + text = format_float_or_int_string(text) + leaf.value = text diff --git a/contrib/python/black/black/output.py b/contrib/python/black/black/output.py new file mode 100644 index 00000000000..0dbd74e5e22 --- /dev/null +++ b/contrib/python/black/black/output.py @@ -0,0 +1,122 @@ +"""Nice output for Black. + +The double calls are for patching purposes in tests. +""" + +import json +import re +import tempfile +from typing import Any, Optional + +from click import echo, style +from mypy_extensions import mypyc_attr + + +@mypyc_attr(patchable=True) +def _out(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + if message is not None: + if "bold" not in styles: + styles["bold"] = True + message = style(message, **styles) + echo(message, nl=nl, err=True) + + +@mypyc_attr(patchable=True) +def _err(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + if message is not None: + if "fg" not in styles: + styles["fg"] = "red" + message = style(message, **styles) + echo(message, nl=nl, err=True) + + +@mypyc_attr(patchable=True) +def out(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + _out(message, nl=nl, **styles) + + +def err(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + _err(message, nl=nl, **styles) + + +def ipynb_diff(a: str, b: str, a_name: str, b_name: str) -> str: + """Return a unified diff string between each cell in notebooks `a` and `b`.""" + a_nb = json.loads(a) + b_nb = json.loads(b) + diff_lines = [ + diff( + "".join(a_nb["cells"][cell_number]["source"]) + "\n", + "".join(b_nb["cells"][cell_number]["source"]) + "\n", + f"{a_name}:cell_{cell_number}", + f"{b_name}:cell_{cell_number}", + ) + for cell_number, cell in enumerate(a_nb["cells"]) + if cell["cell_type"] == "code" + ] + return "".join(diff_lines) + + +_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + + +def _splitlines_no_ff(source: str) -> list[str]: + """Split a string into lines ignoring form feed and other chars. + + This mimics how the Python parser splits source code. + + A simplified version of the function with the same name in Lib/ast.py + """ + result = [match[0] for match in _line_pattern.finditer(source)] + if result[-1] == "": + result.pop(-1) + return result + + +def diff(a: str, b: str, a_name: str, b_name: str) -> str: + """Return a unified diff string between strings `a` and `b`.""" + import difflib + + a_lines = _splitlines_no_ff(a) + b_lines = _splitlines_no_ff(b) + diff_lines = [] + for line in difflib.unified_diff( + a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5 + ): + # Work around https://bugs.python.org/issue2142 + # See: + # https://www.gnu.org/software/diffutils/manual/html_node/Incomplete-Lines.html + if line[-1] == "\n": + diff_lines.append(line) + else: + diff_lines.append(line + "\n") + diff_lines.append("\\ No newline at end of file\n") + return "".join(diff_lines) + + +def color_diff(contents: str) -> str: + """Inject the ANSI color codes to the diff.""" + lines = contents.split("\n") + for i, line in enumerate(lines): + if line.startswith("+++") or line.startswith("---"): + line = "\033[1m" + line + "\033[0m" # bold, reset + elif line.startswith("@@"): + line = "\033[36m" + line + "\033[0m" # cyan, reset + elif line.startswith("+"): + line = "\033[32m" + line + "\033[0m" # green, reset + elif line.startswith("-"): + line = "\033[31m" + line + "\033[0m" # red, reset + lines[i] = line + return "\n".join(lines) + + +@mypyc_attr(patchable=True) +def dump_to_file(*output: str, ensure_final_newline: bool = True) -> str: + """Dump `output` to a temporary file. Return path to the file.""" + with tempfile.NamedTemporaryFile( + mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8" + ) as f: + for lines in output: + f.write(lines) + if ensure_final_newline and lines and lines[-1] != "\n": + f.write("\n") + return f.name diff --git a/contrib/python/black/black/parsing.py b/contrib/python/black/black/parsing.py new file mode 100644 index 00000000000..0019b0c006a --- /dev/null +++ b/contrib/python/black/black/parsing.py @@ -0,0 +1,252 @@ +""" +Parse Python code and perform AST validation. +""" + +import ast +import sys +import warnings +from collections.abc import Collection, Iterator + +from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature +from black.nodes import syms +from blib2to3 import pygram +from blib2to3.pgen2 import driver +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.parse import ParseError +from blib2to3.pgen2.tokenize import TokenError +from blib2to3.pytree import Leaf, Node + + +class InvalidInput(ValueError): + """Raised when input source code fails all parse attempts.""" + + +def get_grammars(target_versions: set[TargetVersion]) -> list[Grammar]: + if not target_versions: + # No target_version specified, so try all grammars. + return [ + # Python 3.7-3.9 + pygram.python_grammar_async_keywords, + # Python 3.0-3.6 + pygram.python_grammar, + # Python 3.10+ + pygram.python_grammar_soft_keywords, + ] + + grammars = [] + # If we have to parse both, try to parse async as a keyword first + if not supports_feature( + target_versions, Feature.ASYNC_IDENTIFIERS + ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING): + # Python 3.7-3.9 + grammars.append(pygram.python_grammar_async_keywords) + if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): + # Python 3.0-3.6 + grammars.append(pygram.python_grammar) + if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions): + # Python 3.10+ + grammars.append(pygram.python_grammar_soft_keywords) + + # At least one of the above branches must have been taken, because every Python + # version has exactly one of the two 'ASYNC_*' flags + return grammars + + +def lib2to3_parse( + src_txt: str, target_versions: Collection[TargetVersion] = () +) -> Node: + """Given a string with source, return the lib2to3 Node.""" + if not src_txt.endswith("\n"): + src_txt += "\n" + + grammars = get_grammars(set(target_versions)) + if target_versions: + max_tv = max(target_versions, key=lambda tv: tv.value) + tv_str = f" for target version {max_tv.pretty()}" + else: + tv_str = "" + + errors = {} + for grammar in grammars: + drv = driver.Driver(grammar) + try: + result = drv.parse_string(src_txt, True) + break + + except ParseError as pe: + lineno, column = pe.context[1] + lines = src_txt.splitlines() + try: + faulty_line = lines[lineno - 1] + except IndexError: + faulty_line = "<line number missing in source>" + errors[grammar.version] = InvalidInput( + f"Cannot parse{tv_str}: {lineno}:{column}: {faulty_line}" + ) + + except TokenError as te: + # In edge cases these are raised; and typically don't have a "faulty_line". + lineno, column = te.args[1] + errors[grammar.version] = InvalidInput( + f"Cannot parse{tv_str}: {lineno}:{column}: {te.args[0]}" + ) + + else: + # Choose the latest version when raising the actual parsing error. + assert len(errors) >= 1 + exc = errors[max(errors)] + raise exc from None + + if isinstance(result, Leaf): + result = Node(syms.file_input, [result]) + return result + + +def matches_grammar(src_txt: str, grammar: Grammar) -> bool: + drv = driver.Driver(grammar) + try: + drv.parse_string(src_txt, True) + except (ParseError, TokenError, IndentationError): + return False + else: + return True + + +def lib2to3_unparse(node: Node) -> str: + """Given a lib2to3 node, return its string representation.""" + code = str(node) + return code + + +class ASTSafetyError(Exception): + """Raised when Black's generated code is not equivalent to the old AST.""" + + +def _parse_single_version( + src: str, version: tuple[int, int], *, type_comments: bool +) -> ast.AST: + filename = "<unknown>" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + warnings.simplefilter("ignore", DeprecationWarning) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) + + +def parse_ast(src: str) -> ast.AST: + # TODO: support Python 4+ ;) + versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)] + + first_error = "" + for version in sorted(versions, reverse=True): + try: + return _parse_single_version(src, version, type_comments=True) + except SyntaxError as e: + if not first_error: + first_error = str(e) + + # Try to parse without type comments + for version in sorted(versions, reverse=True): + try: + return _parse_single_version(src, version, type_comments=False) + except SyntaxError: + pass + + raise SyntaxError(first_error) + + +def _normalize(lineend: str, value: str) -> str: + # To normalize, we strip any leading and trailing space from + # each line... + stripped: list[str] = [i.strip() for i in value.splitlines()] + normalized = lineend.join(stripped) + # ...and remove any blank lines at the beginning and end of + # the whole string + return normalized.strip() + + +def stringify_ast(node: ast.AST) -> Iterator[str]: + """Simple visitor generating strings to compare ASTs by content.""" + return _stringify_ast(node, []) + + +def _stringify_ast_with_new_parent( + node: ast.AST, parent_stack: list[ast.AST], new_parent: ast.AST +) -> Iterator[str]: + parent_stack.append(new_parent) + yield from _stringify_ast(node, parent_stack) + parent_stack.pop() + + +def _stringify_ast(node: ast.AST, parent_stack: list[ast.AST]) -> Iterator[str]: + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.kind == "u" + ): + # It's a quirk of history that we strip the u prefix over here. We used to + # rewrite the AST nodes for Python version compatibility and we never copied + # over the kind + node.kind = None + + yield f"{' ' * len(parent_stack)}{node.__class__.__name__}(" + + for field in sorted(node._fields): # noqa: F402 + # TypeIgnore has only one field 'lineno' which breaks this comparison + if isinstance(node, ast.TypeIgnore): + break + + try: + value: object = getattr(node, field) + except AttributeError: + continue + + yield f"{' ' * (len(parent_stack) + 1)}{field}=" + + if isinstance(value, list): + for item in value: + # Ignore nested tuples within del statements, because we may insert + # parentheses and they change the AST. + if ( + field == "targets" + and isinstance(node, ast.Delete) + and isinstance(item, ast.Tuple) + ): + for elt in item.elts: + yield from _stringify_ast_with_new_parent( + elt, parent_stack, node + ) + + elif isinstance(item, ast.AST): + yield from _stringify_ast_with_new_parent(item, parent_stack, node) + + elif isinstance(value, ast.AST): + yield from _stringify_ast_with_new_parent(value, parent_stack, node) + + else: + normalized: object + if ( + isinstance(node, ast.Constant) + and field == "value" + and isinstance(value, str) + and len(parent_stack) >= 2 + # Any standalone string, ideally this would + # exactly match black.nodes.is_docstring + and isinstance(parent_stack[-1], ast.Expr) + ): + # Constant strings may be indented across newlines, if they are + # docstrings; fold spaces after newlines when comparing. Similarly, + # trailing and leading space may be removed. + normalized = _normalize("\n", value) + elif field == "type_comment" and isinstance(value, str): + # Trailing whitespace in type comments is removed. + normalized = value.rstrip() + else: + normalized = value + yield ( + f"{' ' * (len(parent_stack) + 1)}{normalized!r}, #" + f" {value.__class__.__name__}" + ) + + yield f"{' ' * len(parent_stack)}) # /{node.__class__.__name__}" diff --git a/contrib/python/black/black/py.typed b/contrib/python/black/black/py.typed new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/black/black/py.typed diff --git a/contrib/python/black/black/ranges.py b/contrib/python/black/black/ranges.py new file mode 100644 index 00000000000..90649137d2e --- /dev/null +++ b/contrib/python/black/black/ranges.py @@ -0,0 +1,522 @@ +"""Functions related to Black's formatting by line ranges feature.""" + +import difflib +from collections.abc import Collection, Iterator, Sequence +from dataclasses import dataclass +from typing import Union + +from black.nodes import ( + LN, + STANDALONE_COMMENT, + Leaf, + Node, + Visitor, + first_leaf, + furthest_ancestor_with_last_leaf, + last_leaf, + syms, +) +from blib2to3.pgen2.token import ASYNC, NEWLINE + + +def parse_line_ranges(line_ranges: Sequence[str]) -> list[tuple[int, int]]: + lines: list[tuple[int, int]] = [] + for lines_str in line_ranges: + parts = lines_str.split("-") + if len(parts) != 2: + raise ValueError( + "Incorrect --line-ranges format, expect 'START-END', found" + f" {lines_str!r}" + ) + try: + start = int(parts[0]) + end = int(parts[1]) + except ValueError: + raise ValueError( + "Incorrect --line-ranges value, expect integer ranges, found" + f" {lines_str!r}" + ) from None + else: + lines.append((start, end)) + return lines + + +def is_valid_line_range(lines: tuple[int, int]) -> bool: + """Returns whether the line range is valid.""" + return not lines or lines[0] <= lines[1] + + +def sanitized_lines( + lines: Collection[tuple[int, int]], src_contents: str +) -> Collection[tuple[int, int]]: + """Returns the valid line ranges for the given source. + + This removes ranges that are entirely outside the valid lines. + + Other ranges are normalized so that the start values are at least 1 and the + end values are at most the (1-based) index of the last source line. + """ + if not src_contents: + return [] + good_lines = [] + src_line_count = src_contents.count("\n") + if not src_contents.endswith("\n"): + src_line_count += 1 + for start, end in lines: + if start > src_line_count: + continue + # line-ranges are 1-based + start = max(start, 1) + if end < start: + continue + end = min(end, src_line_count) + good_lines.append((start, end)) + return good_lines + + +def adjusted_lines( + lines: Collection[tuple[int, int]], + original_source: str, + modified_source: str, +) -> list[tuple[int, int]]: + """Returns the adjusted line ranges based on edits from the original code. + + This computes the new line ranges by diffing original_source and + modified_source, and adjust each range based on how the range overlaps with + the diffs. + + Note the diff can contain lines outside of the original line ranges. This can + happen when the formatting has to be done in adjacent to maintain consistent + local results. For example: + + 1. def my_func(arg1, arg2, + 2. arg3,): + 3. pass + + If it restricts to line 2-2, it can't simply reformat line 2, it also has + to reformat line 1: + + 1. def my_func( + 2. arg1, + 3. arg2, + 4. arg3, + 5. ): + 6. pass + + In this case, we will expand the line ranges to also include the whole diff + block. + + Args: + lines: a collection of line ranges. + original_source: the original source. + modified_source: the modified source. + """ + lines_mappings = _calculate_lines_mappings(original_source, modified_source) + + new_lines = [] + # Keep an index of the current search. Since the lines and lines_mappings are + # sorted, this makes the search complexity linear. + current_mapping_index = 0 + for start, end in sorted(lines): + start_mapping_index = _find_lines_mapping_index( + start, + lines_mappings, + current_mapping_index, + ) + end_mapping_index = _find_lines_mapping_index( + end, + lines_mappings, + start_mapping_index, + ) + current_mapping_index = start_mapping_index + if start_mapping_index >= len(lines_mappings) or end_mapping_index >= len( + lines_mappings + ): + # Protect against invalid inputs. + continue + start_mapping = lines_mappings[start_mapping_index] + end_mapping = lines_mappings[end_mapping_index] + if start_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_start = start_mapping.modified_start + else: + new_start = ( + start - start_mapping.original_start + start_mapping.modified_start + ) + if end_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_end = end_mapping.modified_end + else: + new_end = end - end_mapping.original_start + end_mapping.modified_start + new_range = (new_start, new_end) + if is_valid_line_range(new_range): + new_lines.append(new_range) + return new_lines + + +def convert_unchanged_lines(src_node: Node, lines: Collection[tuple[int, int]]) -> None: + """Converts unchanged lines to STANDALONE_COMMENT. + + The idea is similar to how `# fmt: on/off` is implemented. It also converts the + nodes between those markers as a single `STANDALONE_COMMENT` leaf node with + the unformatted code as its value. `STANDALONE_COMMENT` is a "fake" token + that will be formatted as-is with its prefix normalized. + + Here we perform two passes: + + 1. Visit the top-level statements, and convert them to a single + `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some + of the top-level statements aren't changed. + 2. Convert unchanged "unwrapped lines" to `STANDALONE_COMMENT` nodes line by + line. "unwrapped lines" are divided by the `NEWLINE` token. e.g. a + multi-line statement is *one* "unwrapped line" that ends with `NEWLINE`, + even though this statement itself can span multiple lines, and the + tokenizer only sees the last '\n' as the `NEWLINE` token. + + NOTE: During pass (2), comment prefixes and indentations are ALWAYS + normalized even when the lines aren't changed. This is fixable by moving + more formatting to pass (1). However, it's hard to get it correct when + incorrect indentations are used. So we defer this to future optimizations. + """ + lines_set: set[int] = set() + for start, end in lines: + lines_set.update(range(start, end + 1)) + visitor = _TopLevelStatementsVisitor(lines_set) + _ = list(visitor.visit(src_node)) # Consume all results. + _convert_unchanged_line_by_line(src_node, lines_set) + + +def _contains_standalone_comment(node: LN) -> bool: + if isinstance(node, Leaf): + return node.type == STANDALONE_COMMENT + else: + for child in node.children: + if _contains_standalone_comment(child): + return True + return False + + +class _TopLevelStatementsVisitor(Visitor[None]): + """ + A node visitor that converts unchanged top-level statements to + STANDALONE_COMMENT. + + This is used in addition to _convert_unchanged_line_by_line, to + speed up formatting when there are unchanged top-level + classes/functions/statements. + """ + + def __init__(self, lines_set: set[int]): + self._lines_set = lines_set + + def visit_simple_stmt(self, node: Node) -> Iterator[None]: + # This is only called for top-level statements, since `visit_suite` + # won't visit its children nodes. + yield from [] + newline_leaf = last_leaf(node) + if not newline_leaf: + return + assert ( + newline_leaf.type == NEWLINE + ), f"Unexpectedly found leaf.type={newline_leaf.type}" + # We need to find the furthest ancestor with the NEWLINE as the last + # leaf, since a `suite` can simply be a `simple_stmt` when it puts + # its body on the same line. Example: `if cond: pass`. + ancestor = furthest_ancestor_with_last_leaf(newline_leaf) + if not _get_line_range(ancestor).intersection(self._lines_set): + _convert_node_to_standalone_comment(ancestor) + + def visit_suite(self, node: Node) -> Iterator[None]: + yield from [] + # If there is a STANDALONE_COMMENT node, it means parts of the node tree + # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't + # be simply converted by calling str(node). So we just don't convert + # here. + if _contains_standalone_comment(node): + return + # Find the semantic parent of this suite. For `async_stmt` and + # `async_funcdef`, the ASYNC token is defined on a separate level by the + # grammar. + semantic_parent = node.parent + if semantic_parent is not None: + if ( + semantic_parent.prev_sibling is not None + and semantic_parent.prev_sibling.type == ASYNC + ): + semantic_parent = semantic_parent.parent + if semantic_parent is not None and not _get_line_range( + semantic_parent + ).intersection(self._lines_set): + _convert_node_to_standalone_comment(semantic_parent) + + +def _convert_unchanged_line_by_line(node: Node, lines_set: set[int]) -> None: + """Converts unchanged to STANDALONE_COMMENT line by line.""" + for leaf in node.leaves(): + if leaf.type != NEWLINE: + # We only consider "unwrapped lines", which are divided by the NEWLINE + # token. + continue + if leaf.parent and leaf.parent.type == syms.match_stmt: + # The `suite` node is defined as: + # match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT + # Here we need to check `subject_expr`. The `case_block+` will be + # checked by their own NEWLINEs. + nodes_to_ignore: list[LN] = [] + prev_sibling = leaf.prev_sibling + while prev_sibling: + nodes_to_ignore.insert(0, prev_sibling) + prev_sibling = prev_sibling.prev_sibling + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + elif leaf.parent and leaf.parent.type == syms.suite: + # The `suite` node is defined as: + # suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + # We will check `simple_stmt` and `stmt+` separately against the lines set + parent_sibling = leaf.parent.prev_sibling + nodes_to_ignore = [] + while parent_sibling and not parent_sibling.type == syms.suite: + # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`. + nodes_to_ignore.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` and `async_funcdef` where the ASYNC + # token is on the grandparent node. + grandparent = leaf.parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == ASYNC + ): + nodes_to_ignore.insert(0, grandparent.prev_sibling) + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + else: + ancestor = furthest_ancestor_with_last_leaf(leaf) + # Consider multiple decorators as a whole block, as their + # newlines have different behaviors than the rest of the grammar. + if ( + ancestor.type == syms.decorator + and ancestor.parent + and ancestor.parent.type == syms.decorators + ): + ancestor = ancestor.parent + if not _get_line_range(ancestor).intersection(lines_set): + _convert_node_to_standalone_comment(ancestor) + + +def _convert_node_to_standalone_comment(node: LN) -> None: + """Convert node to STANDALONE_COMMENT by modifying the tree inline.""" + parent = node.parent + if not parent: + return + first = first_leaf(node) + last = last_leaf(node) + if not first or not last: + return + if first is last: + # This can happen on the following edge cases: + # 1. A block of `# fmt: off/on` code except the `# fmt: on` is placed + # on the end of the last line instead of on a new line. + # 2. A single backslash on its own line followed by a comment line. + # Ideally we don't want to format them when not requested, but fixing + # isn't easy. These cases are also badly formatted code, so it isn't + # too bad we reformat them. + return + # The prefix contains comments and indentation whitespaces. They are + # reformatted accordingly to the correct indentation level. + # This also means the indentation will be changed on the unchanged lines, and + # this is actually required to not break incremental reformatting. + prefix = first.prefix + first.prefix = "" + index = node.remove() + if index is not None: + # Remove the '\n', as STANDALONE_COMMENT will have '\n' appended when + # generating the formatted code. + value = str(node)[:-1] + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _convert_nodes_to_standalone_comment(nodes: Sequence[LN], *, newline: Leaf) -> None: + """Convert nodes to STANDALONE_COMMENT by modifying the tree inline.""" + if not nodes: + return + parent = nodes[0].parent + first = first_leaf(nodes[0]) + if not parent or not first: + return + prefix = first.prefix + first.prefix = "" + value = "".join(str(node) for node in nodes) + # The prefix comment on the NEWLINE leaf is the trailing comment of the statement. + if newline.prefix: + value += newline.prefix + newline.prefix = "" + index = nodes[0].remove() + for node in nodes[1:]: + node.remove() + if index is not None: + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _leaf_line_end(leaf: Leaf) -> int: + """Returns the line number of the leaf node's last line.""" + if leaf.type == NEWLINE: + return leaf.lineno + else: + # Leaf nodes like multiline strings can occupy multiple lines. + return leaf.lineno + str(leaf).count("\n") + + +def _get_line_range(node_or_nodes: Union[LN, list[LN]]) -> set[int]: + """Returns the line range of this node or list of nodes.""" + if isinstance(node_or_nodes, list): + nodes = node_or_nodes + if not nodes: + return set() + first = first_leaf(nodes[0]) + last = last_leaf(nodes[-1]) + if first and last: + line_start = first.lineno + line_end = _leaf_line_end(last) + return set(range(line_start, line_end + 1)) + else: + return set() + else: + node = node_or_nodes + if isinstance(node, Leaf): + return set(range(node.lineno, _leaf_line_end(node) + 1)) + else: + first = first_leaf(node) + last = last_leaf(node) + if first and last: + return set(range(first.lineno, _leaf_line_end(last) + 1)) + else: + return set() + + +@dataclass +class _LinesMapping: + """1-based lines mapping from original source to modified source. + + Lines [original_start, original_end] from original source + are mapped to [modified_start, modified_end]. + + The ranges are inclusive on both ends. + """ + + original_start: int + original_end: int + modified_start: int + modified_end: int + # Whether this range corresponds to a changed block, or an unchanged block. + is_changed_block: bool + + +def _calculate_lines_mappings( + original_source: str, + modified_source: str, +) -> Sequence[_LinesMapping]: + """Returns a sequence of _LinesMapping by diffing the sources. + + For example, given the following diff: + import re + - def func(arg1, + - arg2, arg3): + + def func(arg1, arg2, arg3): + pass + It returns the following mappings: + original -> modified + (1, 1) -> (1, 1), is_changed_block=False (the "import re" line) + (2, 3) -> (2, 2), is_changed_block=True (the diff) + (4, 4) -> (3, 3), is_changed_block=False (the "pass" line) + + You can think of this visually as if it brings up a side-by-side diff, and tries + to map the line ranges from the left side to the right side: + + (1, 1)->(1, 1) 1. import re 1. import re + (2, 3)->(2, 2) 2. def func(arg1, 2. def func(arg1, arg2, arg3): + 3. arg2, arg3): + (4, 4)->(3, 3) 4. pass 3. pass + + Args: + original_source: the original source. + modified_source: the modified source. + """ + matcher = difflib.SequenceMatcher( + None, + original_source.splitlines(keepends=True), + modified_source.splitlines(keepends=True), + ) + matching_blocks = matcher.get_matching_blocks() + lines_mappings: list[_LinesMapping] = [] + # matching_blocks is a sequence of "same block of code ranges", see + # https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_matching_blocks + # Each block corresponds to a _LinesMapping with is_changed_block=False, + # and the ranges between two blocks corresponds to a _LinesMapping with + # is_changed_block=True, + # NOTE: matching_blocks is 0-based, but _LinesMapping is 1-based. + for i, block in enumerate(matching_blocks): + if i == 0: + if block.a != 0 or block.b != 0: + lines_mappings.append( + _LinesMapping( + original_start=1, + original_end=block.a, + modified_start=1, + modified_end=block.b, + is_changed_block=False, + ) + ) + else: + previous_block = matching_blocks[i - 1] + lines_mappings.append( + _LinesMapping( + original_start=previous_block.a + previous_block.size + 1, + original_end=block.a, + modified_start=previous_block.b + previous_block.size + 1, + modified_end=block.b, + is_changed_block=True, + ) + ) + if i < len(matching_blocks) - 1: + lines_mappings.append( + _LinesMapping( + original_start=block.a + 1, + original_end=block.a + block.size, + modified_start=block.b + 1, + modified_end=block.b + block.size, + is_changed_block=False, + ) + ) + return lines_mappings + + +def _find_lines_mapping_index( + original_line: int, + lines_mappings: Sequence[_LinesMapping], + start_index: int, +) -> int: + """Returns the original index of the lines mappings for the original line.""" + index = start_index + while index < len(lines_mappings): + mapping = lines_mappings[index] + if mapping.original_start <= original_line <= mapping.original_end: + return index + index += 1 + return index diff --git a/contrib/python/black/black/report.py b/contrib/python/black/black/report.py new file mode 100644 index 00000000000..89899f2f389 --- /dev/null +++ b/contrib/python/black/black/report.py @@ -0,0 +1,107 @@ +""" +Summarize Black runs to users. +""" + +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +from click import style + +from black.output import err, out + + +class Changed(Enum): + NO = 0 + CACHED = 1 + YES = 2 + + +class NothingChanged(UserWarning): + """Raised when reformatted code is the same as source.""" + + +@dataclass +class Report: + """Provides a reformatting counter. Can be rendered with `str(report)`.""" + + check: bool = False + diff: bool = False + quiet: bool = False + verbose: bool = False + change_count: int = 0 + same_count: int = 0 + failure_count: int = 0 + + def done(self, src: Path, changed: Changed) -> None: + """Increment the counter for successful reformatting. Write out a message.""" + if changed is Changed.YES: + reformatted = "would reformat" if self.check or self.diff else "reformatted" + if self.verbose or not self.quiet: + out(f"{reformatted} {src}") + self.change_count += 1 + else: + if self.verbose: + if changed is Changed.NO: + msg = f"{src} already well formatted, good job." + else: + msg = f"{src} wasn't modified on disk since last run." + out(msg, bold=False) + self.same_count += 1 + + def failed(self, src: Path, message: str) -> None: + """Increment the counter for failed reformatting. Write out a message.""" + err(f"error: cannot format {src}: {message}") + self.failure_count += 1 + + def path_ignored(self, path: Path, message: str) -> None: + if self.verbose: + out(f"{path} ignored: {message}", bold=False) + + @property + def return_code(self) -> int: + """Return the exit code that the app should use. + + This considers the current state of changed files and failures: + - if there were any failures, return 123; + - if any files were changed and --check is being used, return 1; + - otherwise return 0. + """ + # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with + # 126 we have special return codes reserved by the shell. + if self.failure_count: + return 123 + + elif self.change_count and self.check: + return 1 + + return 0 + + def __str__(self) -> str: + """Render a color report of the current state. + + Use `click.unstyle` to remove colors. + """ + if self.check or self.diff: + reformatted = "would be reformatted" + unchanged = "would be left unchanged" + failed = "would fail to reformat" + else: + reformatted = "reformatted" + unchanged = "left unchanged" + failed = "failed to reformat" + report = [] + if self.change_count: + s = "s" if self.change_count > 1 else "" + report.append( + style(f"{self.change_count} file{s} ", bold=True, fg="blue") + + style(f"{reformatted}", bold=True) + ) + + if self.same_count: + s = "s" if self.same_count > 1 else "" + report.append(style(f"{self.same_count} file{s} ", fg="blue") + unchanged) + if self.failure_count: + s = "s" if self.failure_count > 1 else "" + report.append(style(f"{self.failure_count} file{s} {failed}", fg="red")) + return ", ".join(report) + "." diff --git a/contrib/python/black/black/resources/__init__.py b/contrib/python/black/black/resources/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/black/black/resources/__init__.py diff --git a/contrib/python/black/black/resources/black.schema.json b/contrib/python/black/black/resources/black.schema.json new file mode 100644 index 00000000000..b9b61489136 --- /dev/null +++ b/contrib/python/black/black/resources/black.schema.json @@ -0,0 +1,147 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/psf/black/blob/main/src/black/resources/black.schema.json", + "$comment": "tool.black table in pyproject.toml", + "type": "object", + "additionalProperties": false, + "properties": { + "code": { + "type": "string", + "description": "Format the code passed in as a string." + }, + "line-length": { + "type": "integer", + "description": "How many characters per line to allow.", + "default": 88 + }, + "target-version": { + "type": "array", + "items": { + "enum": [ + "py33", + "py34", + "py35", + "py36", + "py37", + "py38", + "py39", + "py310", + "py311", + "py312", + "py313" + ] + }, + "description": "Python versions that should be supported by Black's output. You should include all versions that your code supports. By default, Black will infer target versions from the project metadata in pyproject.toml. If this does not yield conclusive results, Black will use per-file auto-detection." + }, + "pyi": { + "type": "boolean", + "description": "Format all input files like typing stubs regardless of file extension. This is useful when piping source on standard input.", + "default": false + }, + "ipynb": { + "type": "boolean", + "description": "Format all input files like Jupyter Notebooks regardless of file extension. This is useful when piping source on standard input.", + "default": false + }, + "python-cell-magics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "When processing Jupyter Notebooks, add the given magic to the list of known python-magics (capture, prun, pypy, python, python3, time, timeit). Useful for formatting cells with custom python magics." + }, + "skip-source-first-line": { + "type": "boolean", + "description": "Skip the first line of the source code.", + "default": false + }, + "skip-string-normalization": { + "type": "boolean", + "description": "Don't normalize string quotes or prefixes.", + "default": false + }, + "skip-magic-trailing-comma": { + "type": "boolean", + "description": "Don't use trailing commas as a reason to split lines.", + "default": false + }, + "preview": { + "type": "boolean", + "description": "Enable potentially disruptive style changes that may be added to Black's main functionality in the next major release.", + "default": false + }, + "unstable": { + "type": "boolean", + "description": "Enable potentially disruptive style changes that have known bugs or are not currently expected to make it into the stable style Black's next major release. Implies --preview.", + "default": false + }, + "enable-unstable-feature": { + "type": "array", + "items": { + "enum": [ + "string_processing", + "hug_parens_with_braces_and_square_brackets", + "wrap_long_dict_values_in_parens", + "multiline_string_handling", + "always_one_newline_after_import" + ] + }, + "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." + }, + "check": { + "type": "boolean", + "description": "Don't write the files back, just return the status. Return code 0 means nothing would change. Return code 1 means some files would be reformatted. Return code 123 means there was an internal error.", + "default": false + }, + "diff": { + "type": "boolean", + "description": "Don't write the files back, just output a diff to indicate what changes Black would've made. They are printed to stdout so capturing them is simple.", + "default": false + }, + "color": { + "type": "boolean", + "description": "Show (or do not show) colored diff. Only applies when --diff is given.", + "default": false + }, + "fast": { + "type": "boolean", + "description": "By default, Black performs an AST safety check after formatting your code. The --fast flag turns off this check and the --safe flag explicitly enables it. [default: --safe]", + "default": false + }, + "required-version": { + "type": "string", + "description": "Require a specific version of Black to be running. This is useful for ensuring that all contributors to your project are using the same version, because different versions of Black may format code a little differently. This option can be set in a configuration file for consistent results across environments." + }, + "exclude": { + "type": "string", + "description": "A regular expression that matches files and directories that should be excluded on recursive searches. An empty value means no paths are excluded. Use forward slashes for directories on all platforms (Windows, too). By default, Black also ignores all paths listed in .gitignore. Changing this value will override all default exclusions. [default: /(\\.direnv|\\.eggs|\\.git|\\.hg|\\.ipynb_checkpoints|\\.mypy_cache|\\.nox|\\.pytest_cache|\\.ruff_cache|\\.tox|\\.svn|\\.venv|\\.vscode|__pypackages__|_build|buck-out|build|dist|venv)/]" + }, + "extend-exclude": { + "type": "string", + "description": "Like --exclude, but adds additional files and directories on top of the default values instead of overriding them." + }, + "force-exclude": { + "type": "string", + "description": "Like --exclude, but files and directories matching this regex will be excluded even when they are passed explicitly as arguments. This is useful when invoking Black programmatically on changed files, such as in a pre-commit hook or editor plugin." + }, + "include": { + "type": "string", + "description": "A regular expression that matches files and directories that should be included on recursive searches. An empty value means all files are included regardless of the name. Use forward slashes for directories on all platforms (Windows, too). Overrides all exclusions, including from .gitignore and command line options.", + "default": "(\\.pyi?|\\.ipynb)$" + }, + "workers": { + "type": "integer", + "description": "When Black formats multiple files, it may use a process pool to speed up formatting. This option controls the number of parallel workers. This can also be specified via the BLACK_NUM_WORKERS environment variable. Defaults to the number of CPUs in the system." + }, + "quiet": { + "type": "boolean", + "description": "Stop emitting all non-critical output. Error messages will still be emitted (which can silenced by 2>/dev/null).", + "default": false + }, + "verbose": { + "type": "boolean", + "description": "Emit messages about files that were not changed or were ignored due to exclusion patterns. If Black is using a configuration file, a message detailing which one it is using will be emitted.", + "default": false + } + } +} diff --git a/contrib/python/black/black/rusty.py b/contrib/python/black/black/rusty.py new file mode 100644 index 00000000000..ebd4c052d1f --- /dev/null +++ b/contrib/python/black/black/rusty.py @@ -0,0 +1,28 @@ +"""An error-handling model influenced by that used by the Rust programming language + +See https://doc.rust-lang.org/book/ch09-00-error-handling.html. +""" + +from typing import Generic, TypeVar, Union + +T = TypeVar("T") +E = TypeVar("E", bound=Exception) + + +class Ok(Generic[T]): + def __init__(self, value: T) -> None: + self._value = value + + def ok(self) -> T: + return self._value + + +class Err(Generic[E]): + def __init__(self, e: E) -> None: + self._e = e + + def err(self) -> E: + return self._e + + +Result = Union[Ok[T], Err[E]] diff --git a/contrib/python/black/black/schema.py b/contrib/python/black/black/schema.py new file mode 100644 index 00000000000..f534dbb028d --- /dev/null +++ b/contrib/python/black/black/schema.py @@ -0,0 +1,15 @@ +import importlib.resources +import json +from typing import Any + + +def get_schema(tool_name: str = "black") -> Any: + """Get the stored complete schema for black's settings.""" + assert tool_name == "black", "Only black is supported." + + pkg = "black.resources" + fname = "black.schema.json" + + schema = importlib.resources.files(pkg).joinpath(fname) + with schema.open(encoding="utf-8") as f: + return json.load(f) diff --git a/contrib/python/black/black/strings.py b/contrib/python/black/black/strings.py new file mode 100644 index 00000000000..a3018990ee8 --- /dev/null +++ b/contrib/python/black/black/strings.py @@ -0,0 +1,389 @@ +""" +Simple formatting on strings. Further string formatting code is in trans.py. +""" + +import re +import sys +from functools import lru_cache +from re import Match, Pattern +from typing import Final + +from black._width_table import WIDTH_TABLE +from blib2to3.pytree import Leaf + +STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. +STRING_PREFIX_RE: Final = re.compile( + r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL +) +UNICODE_ESCAPE_RE: Final = re.compile( + r"(?P<backslashes>\\+)(?P<body>" + r"(u(?P<u>[a-fA-F0-9]{4}))" # Character with 16-bit hex value xxxx + r"|(U(?P<U>[a-fA-F0-9]{8}))" # Character with 32-bit hex value xxxxxxxx + r"|(x(?P<x>[a-fA-F0-9]{2}))" # Character with hex value hh + r"|(N\{(?P<N>[a-zA-Z0-9 \-]{2,})\})" # Character named name in the Unicode database + r")", + re.VERBOSE, +) + + +def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str: + """Replace `regex` with `replacement` twice on `original`. + + This is used by string normalization to perform replaces on + overlapping matches. + """ + return regex.sub(replacement, regex.sub(replacement, original)) + + +def has_triple_quotes(string: str) -> bool: + """ + Returns: + True iff @string starts with three quotation characters. + """ + raw_string = string.lstrip(STRING_PREFIX_CHARS) + return raw_string[:3] in {'"""', "'''"} + + +def lines_with_leading_tabs_expanded(s: str) -> list[str]: + """ + Splits string into lines and expands only leading tabs (following the normal + Python rules) + """ + lines = [] + for line in s.splitlines(): + stripped_line = line.lstrip() + if not stripped_line or stripped_line == line: + lines.append(line) + else: + prefix_length = len(line) - len(stripped_line) + prefix = line[:prefix_length].expandtabs() + lines.append(prefix + stripped_line) + if s.endswith("\n"): + lines.append("") + return lines + + +def fix_multiline_docstring(docstring: str, prefix: str) -> str: + # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation + assert docstring, "INTERNAL ERROR: Multiline docstrings cannot be empty" + lines = lines_with_leading_tabs_expanded(docstring) + # Determine minimum indentation (first line doesn't count): + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + last_line_idx = len(lines) - 2 + for i, line in enumerate(lines[1:]): + stripped_line = line[indent:].rstrip() + if stripped_line or i == last_line_idx: + trimmed.append(prefix + stripped_line) + else: + trimmed.append("") + return "\n".join(trimmed) + + +def get_string_prefix(string: str) -> str: + """ + Pre-conditions: + * assert_is_leaf_string(@string) + + Returns: + @string's prefix (e.g. '', 'r', 'f', or 'rf'). + """ + assert_is_leaf_string(string) + + prefix = "" + prefix_idx = 0 + while string[prefix_idx] in STRING_PREFIX_CHARS: + prefix += string[prefix_idx] + prefix_idx += 1 + + return prefix + + +def assert_is_leaf_string(string: str) -> None: + """ + Checks the pre-condition that @string has the format that you would expect + of `leaf.value` where `leaf` is some Leaf such that `leaf.type == + token.STRING`. A more precise description of the pre-conditions that are + checked are listed below. + + Pre-conditions: + * @string starts with either ', ", <prefix>', or <prefix>" where + `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`. + * @string ends with a quote character (' or "). + + Raises: + AssertionError(...) if the pre-conditions listed above are not + satisfied. + """ + dquote_idx = string.find('"') + squote_idx = string.find("'") + if -1 in [dquote_idx, squote_idx]: + quote_idx = max(dquote_idx, squote_idx) + else: + quote_idx = min(squote_idx, dquote_idx) + + assert ( + 0 <= quote_idx < len(string) - 1 + ), f"{string!r} is missing a starting quote character (' or \")." + assert string[-1] in ( + "'", + '"', + ), f"{string!r} is missing an ending quote character (' or \")." + assert set(string[:quote_idx]).issubset( + set(STRING_PREFIX_CHARS) + ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}." + + +def normalize_string_prefix(s: str) -> str: + """Make all string prefixes lowercase.""" + match = STRING_PREFIX_RE.match(s) + assert match is not None, f"failed to match string {s!r}" + orig_prefix = match.group(1) + new_prefix = ( + orig_prefix.replace("F", "f") + .replace("B", "b") + .replace("U", "") + .replace("u", "") + ) + + # Python syntax guarantees max 2 prefixes and that one of them is "r" + if len(new_prefix) == 2 and "r" != new_prefix[0].lower(): + new_prefix = new_prefix[::-1] + return f"{new_prefix}{match.group(2)}" + + +# Re(gex) does actually cache patterns internally but this still improves +# performance on a long list literal of strings by 5-9% since lru_cache's +# caching overhead is much lower. +@lru_cache(maxsize=64) +def _cached_compile(pattern: str) -> Pattern[str]: + return re.compile(pattern) + + +def normalize_string_quotes(s: str) -> str: + """Prefer double quotes but only if it doesn't cause more escaping. + + Adds or removes backslashes as appropriate. + """ + value = s.lstrip(STRING_PREFIX_CHARS) + if value[:3] == '"""': + return s + + elif value[:3] == "'''": + orig_quote = "'''" + new_quote = '"""' + elif value[0] == '"': + orig_quote = '"' + new_quote = "'" + else: + orig_quote = "'" + new_quote = '"' + first_quote_pos = s.find(orig_quote) + assert first_quote_pos != -1, f"INTERNAL ERROR: Malformed string {s!r}" + + prefix = s[:first_quote_pos] + unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}") + escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}") + escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}") + body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)] + if "r" in prefix.casefold(): + if unescaped_new_quote.search(body): + # There's at least one unescaped new_quote in this raw string + # so converting is impossible + return s + + # Do not introduce or remove backslashes in raw strings + new_body = body + else: + # remove unnecessary escapes + new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body) + if body != new_body: + # Consider the string without unnecessary escapes as the original + body = new_body + s = f"{prefix}{orig_quote}{body}{orig_quote}" + new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body) + new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body) + + if "f" in prefix.casefold(): + matches = re.findall( + r""" + (?:(?<!\{)|^)\{ # start of the string or a non-{ followed by a single { + ([^{].*?) # contents of the brackets except if begins with {{ + \}(?:(?!\})|$) # A } followed by end of the string or a non-} + """, + new_body, + re.VERBOSE, + ) + for m in matches: + if "\\" in str(m): + # Do not introduce backslashes in interpolated expressions + return s + + if new_quote == '"""' and new_body[-1:] == '"': + # edge case: + new_body = new_body[:-1] + '\\"' + orig_escape_count = body.count("\\") + new_escape_count = new_body.count("\\") + if new_escape_count > orig_escape_count: + return s # Do not introduce more escaping + + if new_escape_count == orig_escape_count and orig_quote == '"': + return s # Prefer double quotes + + return f"{prefix}{new_quote}{new_body}{new_quote}" + + +def normalize_fstring_quotes( + quote: str, + middles: list[Leaf], + is_raw_fstring: bool, +) -> tuple[list[Leaf], str]: + """Prefer double quotes but only if it doesn't cause more escaping. + + Adds or removes backslashes as appropriate. + """ + if quote == '"""': + return middles, quote + + elif quote == "'''": + new_quote = '"""' + elif quote == '"': + new_quote = "'" + else: + new_quote = '"' + + unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}") + escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}") + escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){quote}") + if is_raw_fstring: + for middle in middles: + if unescaped_new_quote.search(middle.value): + # There's at least one unescaped new_quote in this raw string + # so converting is impossible + return middles, quote + + # Do not introduce or remove backslashes in raw strings, just use double quote + return middles, '"' + + new_segments = [] + for middle in middles: + segment = middle.value + # remove unnecessary escapes + new_segment = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", segment) + if segment != new_segment: + # Consider the string without unnecessary escapes as the original + middle.value = new_segment + + new_segment = sub_twice(escaped_orig_quote, rf"\1\2{quote}", new_segment) + new_segment = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_segment) + new_segments.append(new_segment) + + if new_quote == '"""' and new_segments[-1].endswith('"'): + # edge case: + new_segments[-1] = new_segments[-1][:-1] + '\\"' + + for middle, new_segment in zip(middles, new_segments): + orig_escape_count = middle.value.count("\\") + new_escape_count = new_segment.count("\\") + + if new_escape_count > orig_escape_count: + return middles, quote # Do not introduce more escaping + + if new_escape_count == orig_escape_count and quote == '"': + return middles, quote # Prefer double quotes + + for middle, new_segment in zip(middles, new_segments): + middle.value = new_segment + + return middles, new_quote + + +def normalize_unicode_escape_sequences(leaf: Leaf) -> None: + """Replace hex codes in Unicode escape sequences with lowercase representation.""" + text = leaf.value + prefix = get_string_prefix(text) + if "r" in prefix.lower(): + return + + def replace(m: Match[str]) -> str: + groups = m.groupdict() + back_slashes = groups["backslashes"] + + if len(back_slashes) % 2 == 0: + return back_slashes + groups["body"] + + if groups["u"]: + # \u + return back_slashes + "u" + groups["u"].lower() + elif groups["U"]: + # \U + return back_slashes + "U" + groups["U"].lower() + elif groups["x"]: + # \x + return back_slashes + "x" + groups["x"].lower() + else: + assert groups["N"], f"Unexpected match: {m}" + # \N{} + return back_slashes + "N{" + groups["N"].upper() + "}" + + leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text) + + +@lru_cache(maxsize=4096) +def char_width(char: str) -> int: + """Return the width of a single character as it would be displayed in a + terminal or editor (which respects Unicode East Asian Width). + + Full width characters are counted as 2, while half width characters are + counted as 1. Also control characters are counted as 0. + """ + table = WIDTH_TABLE + codepoint = ord(char) + highest = len(table) - 1 + lowest = 0 + idx = highest // 2 + while True: + start_codepoint, end_codepoint, width = table[idx] + if codepoint < start_codepoint: + highest = idx - 1 + elif codepoint > end_codepoint: + lowest = idx + 1 + else: + return 0 if width < 0 else width + if highest < lowest: + break + idx = (highest + lowest) // 2 + return 1 + + +def str_width(line_str: str) -> int: + """Return the width of `line_str` as it would be displayed in a terminal + or editor (which respects Unicode East Asian Width). + + You could utilize this function to determine, for example, if a string + is too wide to display in a terminal or editor. + """ + if line_str.isascii(): + # Fast path for a line consisting of only ASCII characters + return len(line_str) + return sum(map(char_width, line_str)) + + +def count_chars_in_width(line_str: str, max_width: int) -> int: + """Count the number of characters in `line_str` that would fit in a + terminal or editor of `max_width` (which respects Unicode East Asian + Width). + """ + total_width = 0 + for i, char in enumerate(line_str): + width = char_width(char) + if width + total_width > max_width: + return i + total_width += width + return len(line_str) diff --git a/contrib/python/black/black/trans.py b/contrib/python/black/black/trans.py new file mode 100644 index 00000000000..fabc7051108 --- /dev/null +++ b/contrib/python/black/black/trans.py @@ -0,0 +1,2509 @@ +""" +String transformers that can split and merge strings. +""" + +import re +from abc import ABC, abstractmethod +from collections import defaultdict +from collections.abc import Callable, Collection, Iterable, Iterator, Sequence +from dataclasses import dataclass +from typing import Any, ClassVar, Final, Literal, Optional, TypeVar, Union + +from mypy_extensions import trait + +from black.comments import contains_pragma_comment +from black.lines import Line, append_leaves +from black.mode import Feature, Mode +from black.nodes import ( + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + is_empty_lpar, + is_empty_par, + is_empty_rpar, + is_part_of_annotation, + parent_type, + replace_child, + syms, +) +from black.rusty import Err, Ok, Result +from black.strings import ( + assert_is_leaf_string, + count_chars_in_width, + get_string_prefix, + has_triple_quotes, + normalize_string_quotes, + str_width, +) +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node + + +class CannotTransform(Exception): + """Base class for errors raised by Transformers.""" + + +# types +T = TypeVar("T") +LN = Union[Leaf, Node] +Transformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]] +Index = int +NodeType = int +ParserState = int +StringID = int +TResult = Result[T, CannotTransform] # (T)ransform Result +TMatchResult = TResult[list[Index]] + +SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops + + +def TErr(err_msg: str) -> Err[CannotTransform]: + """(T)ransform Err + + Convenience function used when working with the TResult type. + """ + cant_transform = CannotTransform(err_msg) + return Err(cant_transform) + + +def hug_power_op( + line: Line, features: Collection[Feature], mode: Mode +) -> Iterator[Line]: + """A transformer which normalizes spacing around power operators.""" + + # Performance optimization to avoid unnecessary Leaf clones and other ops. + for leaf in line.leaves: + if leaf.type == token.DOUBLESTAR: + break + else: + raise CannotTransform("No doublestar token was found in the line.") + + def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool: + # Brackets and parentheses indicate calls, subscripts, etc. ... + # basically stuff that doesn't count as "simple". Only a NAME lookup + # or dotted lookup (eg. NAME.NAME) is OK. + if kind == -1: + return handle_is_simple_look_up_prev(line, index, {token.RPAR, token.RSQB}) + else: + return handle_is_simple_lookup_forward( + line, index, {token.LPAR, token.LSQB} + ) + + def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: + # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple + # lookup (see above), with or without a preceding unary operator. + start = line.leaves[index] + if start.type in {token.NAME, token.NUMBER}: + return is_simple_lookup(index, kind) + + if start.type in {token.PLUS, token.MINUS, token.TILDE}: + if line.leaves[index + 1].type in {token.NAME, token.NUMBER}: + # kind is always one as bases with a preceding unary op will be checked + # for simplicity starting from the next token (so it'll hit the check + # above). + return is_simple_lookup(index + 1, kind=1) + + return False + + new_line = line.clone() + should_hug = False + for idx, leaf in enumerate(line.leaves): + new_leaf = leaf.clone() + if should_hug: + new_leaf.prefix = "" + should_hug = False + + should_hug = ( + (0 < idx < len(line.leaves) - 1) + and leaf.type == token.DOUBLESTAR + and is_simple_operand(idx - 1, kind=-1) + and line.leaves[idx - 1].value != "lambda" + and is_simple_operand(idx + 1, kind=1) + ) + if should_hug: + new_leaf.prefix = "" + + # We have to be careful to make a new line properly: + # - bracket related metadata must be maintained (handled by Line.append) + # - comments need to copied over, updating the leaf IDs they're attached to + new_line.append(new_leaf, preformatted=True) + for comment_leaf in line.comments_after(leaf): + new_line.append(comment_leaf, preformatted=True) + + yield new_line + + +def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool: + """ + Handling the determination of is_simple_lookup for the lines prior to the doublestar + token. This is required because of the need to isolate the chained expression + to determine the bracket or parenthesis belong to the single expression. + """ + contains_disallowed = False + chain = [] + + while 0 <= index < len(line.leaves): + current = line.leaves[index] + chain.append(current) + if not contains_disallowed and current.type in disallowed: + contains_disallowed = True + if not is_expression_chained(chain): + return not contains_disallowed + + index -= 1 + + return True + + +def handle_is_simple_lookup_forward( + line: Line, index: int, disallowed: set[int] +) -> bool: + """ + Handling decision is_simple_lookup for the lines behind the doublestar token. + This function is simplified to keep consistent with the prior logic and the forward + case are more straightforward and do not need to care about chained expressions. + """ + while 0 <= index < len(line.leaves): + current = line.leaves[index] + if current.type in disallowed: + return False + if current.type not in {token.NAME, token.DOT} or ( + current.type == token.NAME and current.value == "for" + ): + # If the current token isn't disallowed, we'll assume this is simple as + # only the disallowed tokens are semantically attached to this lookup + # expression we're checking. Also, stop early if we hit the 'for' bit + # of a comprehension. + return True + + index += 1 + + return True + + +def is_expression_chained(chained_leaves: list[Leaf]) -> bool: + """ + Function to determine if the variable is a chained call. + (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call) + """ + if len(chained_leaves) < 2: + return True + + current_leaf = chained_leaves[-1] + past_leaf = chained_leaves[-2] + + if past_leaf.type == token.NAME: + return current_leaf.type in {token.DOT} + elif past_leaf.type in {token.RPAR, token.RSQB}: + return current_leaf.type in {token.RSQB, token.RPAR} + elif past_leaf.type in {token.LPAR, token.LSQB}: + return current_leaf.type in {token.NAME, token.LPAR, token.LSQB} + else: + return False + + +class StringTransformer(ABC): + """ + An implementation of the Transformer protocol that relies on its + subclasses overriding the template methods `do_match(...)` and + `do_transform(...)`. + + This Transformer works exclusively on strings (for example, by merging + or splitting them). + + The following sections can be found among the docstrings of each concrete + StringTransformer subclass. + + Requirements: + Which requirements must be met of the given Line for this + StringTransformer to be applied? + + Transformations: + If the given Line meets all of the above requirements, which string + transformations can you expect to be applied to it by this + StringTransformer? + + Collaborations: + What contractual agreements does this StringTransformer have with other + StringTransfomers? Such collaborations should be eliminated/minimized + as much as possible. + """ + + __name__: Final = "StringTransformer" + + # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with + # `abc.ABC`. + def __init__(self, line_length: int, normalize_strings: bool) -> None: + self.line_length = line_length + self.normalize_strings = normalize_strings + + @abstractmethod + def do_match(self, line: Line) -> TMatchResult: + """ + Returns: + * Ok(string_indices) such that for each index, `line.leaves[index]` + is our target string if a match was able to be made. For + transformers that don't result in more lines (e.g. StringMerger, + StringParenStripper), multiple matches and transforms are done at + once to reduce the complexity. + OR + * Err(CannotTransform), if no match could be made. + """ + + @abstractmethod + def do_transform( + self, line: Line, string_indices: list[int] + ) -> Iterator[TResult[Line]]: + """ + Yields: + * Ok(new_line) where new_line is the new transformed line. + OR + * Err(CannotTransform) if the transformation failed for some reason. The + `do_match(...)` template method should usually be used to reject + the form of the given Line, but in some cases it is difficult to + know whether or not a Line meets the StringTransformer's + requirements until the transformation is already midway. + + Side Effects: + This method should NOT mutate @line directly, but it MAY mutate the + Line's underlying Node structure. (WARNING: If the underlying Node + structure IS altered, then this method should NOT be allowed to + yield an CannotTransform after that point.) + """ + + def __call__( + self, line: Line, _features: Collection[Feature], _mode: Mode + ) -> Iterator[Line]: + """ + StringTransformer instances have a call signature that mirrors that of + the Transformer type. + + Raises: + CannotTransform(...) if the concrete StringTransformer class is unable + to transform @line. + """ + # Optimization to avoid calling `self.do_match(...)` when the line does + # not contain any string. + if not any(leaf.type == token.STRING for leaf in line.leaves): + raise CannotTransform("There are no strings in this line.") + + match_result = self.do_match(line) + + if isinstance(match_result, Err): + cant_transform = match_result.err() + raise CannotTransform( + f"The string transformer {self.__class__.__name__} does not recognize" + " this line as one that it can transform." + ) from cant_transform + + string_indices = match_result.ok() + + for line_result in self.do_transform(line, string_indices): + if isinstance(line_result, Err): + cant_transform = line_result.err() + raise CannotTransform( + "StringTransformer failed while attempting to transform string." + ) from cant_transform + line = line_result.ok() + yield line + + +@dataclass +class CustomSplit: + """A custom (i.e. manual) string split. + + A single CustomSplit instance represents a single substring. + + Examples: + Consider the following string: + ``` + "Hi there friend." + " This is a custom" + f" string {split}." + ``` + + This string will correspond to the following three CustomSplit instances: + ``` + CustomSplit(False, 16) + CustomSplit(False, 17) + CustomSplit(True, 16) + ``` + """ + + has_prefix: bool + break_idx: int + + +@trait +class CustomSplitMapMixin: + """ + This mixin class is used to map merged strings to a sequence of + CustomSplits, which will then be used to re-split the strings iff none of + the resultant substrings go over the configured max line length. + """ + + _Key: ClassVar = tuple[StringID, str] + _CUSTOM_SPLIT_MAP: ClassVar[dict[_Key, tuple[CustomSplit, ...]]] = defaultdict( + tuple + ) + + @staticmethod + def _get_key(string: str) -> "CustomSplitMapMixin._Key": + """ + Returns: + A unique identifier that is used internally to map @string to a + group of custom splits. + """ + return (id(string), string) + + def add_custom_splits( + self, string: str, custom_splits: Iterable[CustomSplit] + ) -> None: + """Custom Split Map Setter Method + + Side Effects: + Adds a mapping from @string to the custom splits @custom_splits. + """ + key = self._get_key(string) + self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits) + + def pop_custom_splits(self, string: str) -> list[CustomSplit]: + """Custom Split Map Getter Method + + Returns: + * A list of the custom splits that are mapped to @string, if any + exist. + OR + * [], otherwise. + + Side Effects: + Deletes the mapping between @string and its associated custom + splits (which are returned to the caller). + """ + key = self._get_key(string) + + custom_splits = self._CUSTOM_SPLIT_MAP[key] + del self._CUSTOM_SPLIT_MAP[key] + + return list(custom_splits) + + def has_custom_splits(self, string: str) -> bool: + """ + Returns: + True iff @string is associated with a set of custom splits. + """ + key = self._get_key(string) + return key in self._CUSTOM_SPLIT_MAP + + +class StringMerger(StringTransformer, CustomSplitMapMixin): + """StringTransformer that merges strings together. + + Requirements: + (A) The line contains adjacent strings such that ALL of the validation checks + listed in StringMerger._validate_msg(...)'s docstring pass. + OR + (B) The line contains a string which uses line continuation backslashes. + + Transformations: + Depending on which of the two requirements above where met, either: + + (A) The string group associated with the target string is merged. + OR + (B) All line-continuation backslashes are removed from the target string. + + Collaborations: + StringMerger provides custom split information to StringSplitter. + """ + + def do_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + string_indices = [] + idx = 0 + while is_valid_index(idx): + leaf = LL[idx] + if ( + leaf.type == token.STRING + and is_valid_index(idx + 1) + and LL[idx + 1].type == token.STRING + ): + # Let's check if the string group contains an inline comment + # If we have a comment inline, we don't merge the strings + contains_comment = False + i = idx + while is_valid_index(i): + if LL[i].type != token.STRING: + break + if line.comments_after(LL[i]): + contains_comment = True + break + i += 1 + + if not contains_comment and not is_part_of_annotation(leaf): + string_indices.append(idx) + + # Advance to the next non-STRING leaf. + idx += 2 + while is_valid_index(idx) and LL[idx].type == token.STRING: + idx += 1 + + elif leaf.type == token.STRING and "\\\n" in leaf.value: + string_indices.append(idx) + # Advance to the next non-STRING leaf. + idx += 1 + while is_valid_index(idx) and LL[idx].type == token.STRING: + idx += 1 + + else: + idx += 1 + + if string_indices: + return Ok(string_indices) + else: + return TErr("This line has no strings that need merging.") + + def do_transform( + self, line: Line, string_indices: list[int] + ) -> Iterator[TResult[Line]]: + new_line = line + + rblc_result = self._remove_backslash_line_continuation_chars( + new_line, string_indices + ) + if isinstance(rblc_result, Ok): + new_line = rblc_result.ok() + + msg_result = self._merge_string_group(new_line, string_indices) + if isinstance(msg_result, Ok): + new_line = msg_result.ok() + + if isinstance(rblc_result, Err) and isinstance(msg_result, Err): + msg_cant_transform = msg_result.err() + rblc_cant_transform = rblc_result.err() + cant_transform = CannotTransform( + "StringMerger failed to merge any strings in this line." + ) + + # Chain the errors together using `__cause__`. + msg_cant_transform.__cause__ = rblc_cant_transform + cant_transform.__cause__ = msg_cant_transform + + yield Err(cant_transform) + else: + yield Ok(new_line) + + @staticmethod + def _remove_backslash_line_continuation_chars( + line: Line, string_indices: list[int] + ) -> TResult[Line]: + """ + Merge strings that were split across multiple lines using + line-continuation backslashes. + + Returns: + Ok(new_line), if @line contains backslash line-continuation + characters. + OR + Err(CannotTransform), otherwise. + """ + LL = line.leaves + + indices_to_transform = [] + for string_idx in string_indices: + string_leaf = LL[string_idx] + if ( + string_leaf.type == token.STRING + and "\\\n" in string_leaf.value + and not has_triple_quotes(string_leaf.value) + ): + indices_to_transform.append(string_idx) + + if not indices_to_transform: + return TErr( + "Found no string leaves that contain backslash line continuation" + " characters." + ) + + new_line = line.clone() + new_line.comments = line.comments.copy() + append_leaves(new_line, line, LL) + + for string_idx in indices_to_transform: + new_string_leaf = new_line.leaves[string_idx] + new_string_leaf.value = new_string_leaf.value.replace("\\\n", "") + + return Ok(new_line) + + def _merge_string_group( + self, line: Line, string_indices: list[int] + ) -> TResult[Line]: + """ + Merges string groups (i.e. set of adjacent strings). + + Each index from `string_indices` designates one string group's first + leaf in `line.leaves`. + + Returns: + Ok(new_line), if ALL of the validation checks found in + _validate_msg(...) pass. + OR + Err(CannotTransform), otherwise. + """ + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + # A dict of {string_idx: tuple[num_of_strings, string_leaf]}. + merged_string_idx_dict: dict[int, tuple[int, Leaf]] = {} + for string_idx in string_indices: + vresult = self._validate_msg(line, string_idx) + if isinstance(vresult, Err): + continue + merged_string_idx_dict[string_idx] = self._merge_one_string_group( + LL, string_idx, is_valid_index + ) + + if not merged_string_idx_dict: + return TErr("No string group is merged") + + # Build the final line ('new_line') that this method will later return. + new_line = line.clone() + previous_merged_string_idx = -1 + previous_merged_num_of_strings = -1 + for i, leaf in enumerate(LL): + if i in merged_string_idx_dict: + previous_merged_string_idx = i + previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i] + new_line.append(string_leaf) + + if ( + previous_merged_string_idx + <= i + < previous_merged_string_idx + previous_merged_num_of_strings + ): + for comment_leaf in line.comments_after(LL[i]): + new_line.append(comment_leaf, preformatted=True) + continue + + append_leaves(new_line, line, [leaf]) + + return Ok(new_line) + + def _merge_one_string_group( + self, LL: list[Leaf], string_idx: int, is_valid_index: Callable[[int], bool] + ) -> tuple[int, Leaf]: + """ + Merges one string group where the first string in the group is + `LL[string_idx]`. + + Returns: + A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the + number of strings merged and `leaf` is the newly merged string + to be replaced in the new line. + """ + # If the string group is wrapped inside an Atom node, we must make sure + # to later replace that Atom with our new (merged) string leaf. + atom_node = LL[string_idx].parent + + # We will place BREAK_MARK in between every two substrings that we + # merge. We will then later go through our final result and use the + # various instances of BREAK_MARK we find to add the right values to + # the custom split map. + BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@" + + QUOTE = LL[string_idx].value[-1] + + def make_naked(string: str, string_prefix: str) -> str: + """Strip @string (i.e. make it a "naked" string) + + Pre-conditions: + * assert_is_leaf_string(@string) + + Returns: + A string that is identical to @string except that + @string_prefix has been stripped, the surrounding QUOTE + characters have been removed, and any remaining QUOTE + characters have been escaped. + """ + assert_is_leaf_string(string) + if "f" in string_prefix: + f_expressions = [ + string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces + for span in iter_fexpr_spans(string) + ] + debug_expressions_contain_visible_quotes = any( + re.search(r".*[\'\"].*(?<![!:=])={1}(?!=)(?![^\s:])", expression) + for expression in f_expressions + ) + if not debug_expressions_contain_visible_quotes: + # We don't want to toggle visible quotes in debug f-strings, as + # that would modify the AST + string = _toggle_fexpr_quotes(string, QUOTE) + # After quotes toggling, quotes in expressions won't be escaped + # because quotes can't be reused in f-strings. So we can simply + # let the escaping logic below run without knowing f-string + # expressions. + + RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)" + naked_string = string[len(string_prefix) + 1 : -1] + naked_string = re.sub( + "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string + ) + return naked_string + + # Holds the CustomSplit objects that will later be added to the custom + # split map. + custom_splits = [] + + # Temporary storage for the 'has_prefix' part of the CustomSplit objects. + prefix_tracker = [] + + # Sets the 'prefix' variable. This is the prefix that the final merged + # string will have. + next_str_idx = string_idx + prefix = "" + while ( + not prefix + and is_valid_index(next_str_idx) + and LL[next_str_idx].type == token.STRING + ): + prefix = get_string_prefix(LL[next_str_idx].value).lower() + next_str_idx += 1 + + # The next loop merges the string group. The final string will be + # contained in 'S'. + # + # The following convenience variables are used: + # + # S: string + # NS: naked string + # SS: next string + # NSS: naked next string + S = "" + NS = "" + num_of_strings = 0 + next_str_idx = string_idx + while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING: + num_of_strings += 1 + + SS = LL[next_str_idx].value + next_prefix = get_string_prefix(SS).lower() + + # If this is an f-string group but this substring is not prefixed + # with 'f'... + if "f" in prefix and "f" not in next_prefix: + # Then we must escape any braces contained in this substring. + SS = re.sub(r"(\{|\})", r"\1\1", SS) + + NSS = make_naked(SS, next_prefix) + + has_prefix = bool(next_prefix) + prefix_tracker.append(has_prefix) + + S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE + NS = make_naked(S, prefix) + + next_str_idx += 1 + + # Take a note on the index of the non-STRING leaf. + non_string_idx = next_str_idx + + S_leaf = Leaf(token.STRING, S) + if self.normalize_strings: + S_leaf.value = normalize_string_quotes(S_leaf.value) + + # Fill the 'custom_splits' list with the appropriate CustomSplit objects. + temp_string = S_leaf.value[len(prefix) + 1 : -1] + for has_prefix in prefix_tracker: + mark_idx = temp_string.find(BREAK_MARK) + assert ( + mark_idx >= 0 + ), "Logic error while filling the custom string breakpoint cache." + + temp_string = temp_string[mark_idx + len(BREAK_MARK) :] + breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1 + custom_splits.append(CustomSplit(has_prefix, breakpoint_idx)) + + string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, "")) + + if atom_node is not None: + # If not all children of the atom node are merged (this can happen + # when there is a standalone comment in the middle) ... + if non_string_idx - string_idx < len(atom_node.children): + # We need to replace the old STRING leaves with the new string leaf. + first_child_idx = LL[string_idx].remove() + for idx in range(string_idx + 1, non_string_idx): + LL[idx].remove() + if first_child_idx is not None: + atom_node.insert_child(first_child_idx, string_leaf) + else: + # Else replace the atom node with the new string leaf. + replace_child(atom_node, string_leaf) + + self.add_custom_splits(string_leaf.value, custom_splits) + return num_of_strings, string_leaf + + @staticmethod + def _validate_msg(line: Line, string_idx: int) -> TResult[None]: + """Validate (M)erge (S)tring (G)roup + + Transform-time string validation logic for _merge_string_group(...). + + Returns: + * Ok(None), if ALL validation checks (listed below) pass. + OR + * Err(CannotTransform), if any of the following are true: + - The target string group does not contain ANY stand-alone comments. + - The target string is not in a string group (i.e. it has no + adjacent strings). + - The string group has more than one inline comment. + - The string group has an inline comment that appears to be a pragma. + - The set of all string prefixes in the string group is of + length greater than one and is not equal to {"", "f"}. + - The string group consists of raw strings. + - The string group would merge f-strings with different quote types + and internal quotes. + - The string group is stringified type annotations. We don't want to + process stringified type annotations since pyright doesn't support + them spanning multiple string values. (NOTE: mypy, pytype, pyre do + support them, so we can change if pyright also gains support in the + future. See https://github.com/microsoft/pyright/issues/4359.) + """ + # We first check for "inner" stand-alone comments (i.e. stand-alone + # comments that have a string leaf before them AND after them). + for inc in [1, -1]: + i = string_idx + found_sa_comment = False + is_valid_index = is_valid_index_factory(line.leaves) + while is_valid_index(i) and line.leaves[i].type in [ + token.STRING, + STANDALONE_COMMENT, + ]: + if line.leaves[i].type == STANDALONE_COMMENT: + found_sa_comment = True + elif found_sa_comment: + return TErr( + "StringMerger does NOT merge string groups which contain " + "stand-alone comments." + ) + + i += inc + + QUOTE = line.leaves[string_idx].value[-1] + + num_of_inline_string_comments = 0 + set_of_prefixes = set() + num_of_strings = 0 + for leaf in line.leaves[string_idx:]: + if leaf.type != token.STRING: + # If the string group is trailed by a comma, we count the + # comments trailing the comma to be one of the string group's + # comments. + if leaf.type == token.COMMA and id(leaf) in line.comments: + num_of_inline_string_comments += 1 + break + + if has_triple_quotes(leaf.value): + return TErr("StringMerger does NOT merge multiline strings.") + + num_of_strings += 1 + prefix = get_string_prefix(leaf.value).lower() + if "r" in prefix: + return TErr("StringMerger does NOT merge raw strings.") + + set_of_prefixes.add(prefix) + + if ( + "f" in prefix + and leaf.value[-1] != QUOTE + and ( + "'" in leaf.value[len(prefix) + 1 : -1] + or '"' in leaf.value[len(prefix) + 1 : -1] + ) + ): + return TErr( + "StringMerger does NOT merge f-strings with different quote types" + " and internal quotes." + ) + + if id(leaf) in line.comments: + num_of_inline_string_comments += 1 + if contains_pragma_comment(line.comments[id(leaf)]): + return TErr("Cannot merge strings which have pragma comments.") + + if num_of_strings < 2: + return TErr( + f"Not enough strings to merge (num_of_strings={num_of_strings})." + ) + + if num_of_inline_string_comments > 1: + return TErr( + f"Too many inline string comments ({num_of_inline_string_comments})." + ) + + if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}: + return TErr(f"Too many different prefixes ({set_of_prefixes}).") + + return Ok(None) + + +class StringParenStripper(StringTransformer): + """StringTransformer that strips surrounding parentheses from strings. + + Requirements: + The line contains a string which is surrounded by parentheses and: + - The target string is NOT the only argument to a function call. + - The target string is NOT a "pointless" string. + - The target string is NOT a dictionary value. + - If the target string contains a PERCENT, the brackets are not + preceded or followed by an operator with higher precedence than + PERCENT. + + Transformations: + The parentheses mentioned in the 'Requirements' section are stripped. + + Collaborations: + StringParenStripper has its own inherent usefulness, but it is also + relied on to clean up the parentheses created by StringParenWrapper (in + the event that they are no longer needed). + """ + + def do_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + string_indices = [] + + idx = -1 + while True: + idx += 1 + if idx >= len(LL): + break + leaf = LL[idx] + + # Should be a string... + if leaf.type != token.STRING: + continue + + # If this is a "pointless" string... + if ( + leaf.parent + and leaf.parent.parent + and leaf.parent.parent.type == syms.simple_stmt + ): + continue + + # Should be preceded by a non-empty LPAR... + if ( + not is_valid_index(idx - 1) + or LL[idx - 1].type != token.LPAR + or is_empty_lpar(LL[idx - 1]) + ): + continue + + # That LPAR should NOT be preceded by a colon (which could be a + # dictionary value), function name, or a closing bracket (which + # could be a function returning a function or a list/dictionary + # containing a function)... + if is_valid_index(idx - 2) and ( + LL[idx - 2].type == token.COLON + or LL[idx - 2].type == token.NAME + or LL[idx - 2].type in CLOSING_BRACKETS + ): + continue + + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + next_idx = string_parser.parse(LL, string_idx) + + # if the leaves in the parsed string include a PERCENT, we need to + # make sure the initial LPAR is NOT preceded by an operator with + # higher or equal precedence to PERCENT + if is_valid_index(idx - 2): + # mypy can't quite follow unless we name this + before_lpar = LL[idx - 2] + if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and ( + ( + before_lpar.type + in { + token.STAR, + token.AT, + token.SLASH, + token.DOUBLESLASH, + token.PERCENT, + token.TILDE, + token.DOUBLESTAR, + token.AWAIT, + token.LSQB, + token.LPAR, + } + ) + or ( + # only unary PLUS/MINUS + before_lpar.parent + and before_lpar.parent.type == syms.factor + and (before_lpar.type in {token.PLUS, token.MINUS}) + ) + ): + continue + + # Should be followed by a non-empty RPAR... + if ( + is_valid_index(next_idx) + and LL[next_idx].type == token.RPAR + and not is_empty_rpar(LL[next_idx]) + ): + # That RPAR should NOT be followed by anything with higher + # precedence than PERCENT + if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in { + token.DOUBLESTAR, + token.LSQB, + token.LPAR, + token.DOT, + }: + continue + + string_indices.append(string_idx) + idx = string_idx + while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING: + idx += 1 + + if string_indices: + return Ok(string_indices) + return TErr("This line has no strings wrapped in parens.") + + def do_transform( + self, line: Line, string_indices: list[int] + ) -> Iterator[TResult[Line]]: + LL = line.leaves + + string_and_rpar_indices: list[int] = [] + for string_idx in string_indices: + string_parser = StringParser() + rpar_idx = string_parser.parse(LL, string_idx) + + should_transform = True + for leaf in (LL[string_idx - 1], LL[rpar_idx]): + if line.comments_after(leaf): + # Should not strip parentheses which have comments attached + # to them. + should_transform = False + break + if should_transform: + string_and_rpar_indices.extend((string_idx, rpar_idx)) + + if string_and_rpar_indices: + yield Ok(self._transform_to_new_line(line, string_and_rpar_indices)) + else: + yield Err( + CannotTransform("All string groups have comments attached to them.") + ) + + def _transform_to_new_line( + self, line: Line, string_and_rpar_indices: list[int] + ) -> Line: + LL = line.leaves + + new_line = line.clone() + new_line.comments = line.comments.copy() + + previous_idx = -1 + # We need to sort the indices, since string_idx and its matching + # rpar_idx may not come in order, e.g. in + # `("outer" % ("inner".join(items)))`, the "inner" string's + # string_idx is smaller than "outer" string's rpar_idx. + for idx in sorted(string_and_rpar_indices): + leaf = LL[idx] + lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx + append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx]) + if leaf.type == token.STRING: + string_leaf = Leaf(token.STRING, LL[idx].value) + LL[lpar_or_rpar_idx].remove() # Remove lpar. + replace_child(LL[idx], string_leaf) + new_line.append(string_leaf) + # replace comments + old_comments = new_line.comments.pop(id(LL[idx]), []) + new_line.comments.setdefault(id(string_leaf), []).extend(old_comments) + else: + LL[lpar_or_rpar_idx].remove() # This is a rpar. + + previous_idx = idx + + # Append the leaves after the last idx: + append_leaves(new_line, line, LL[idx + 1 :]) + + return new_line + + +class BaseStringSplitter(StringTransformer): + """ + Abstract class for StringTransformers which transform a Line's strings by splitting + them or placing them on their own lines where necessary to avoid going over + the configured line length. + + Requirements: + * The target string value is responsible for the line going over the + line length limit. It follows that after all of black's other line + split methods have been exhausted, this line (or one of the resulting + lines after all line splits are performed) would still be over the + line_length limit unless we split this string. + AND + + * The target string is NOT a "pointless" string (i.e. a string that has + no parent or siblings). + AND + + * The target string is not followed by an inline comment that appears + to be a pragma. + AND + + * The target string is not a multiline (i.e. triple-quote) string. + """ + + STRING_OPERATORS: Final = [ + token.EQEQUAL, + token.GREATER, + token.GREATEREQUAL, + token.LESS, + token.LESSEQUAL, + token.NOTEQUAL, + token.PERCENT, + token.PLUS, + token.STAR, + ] + + @abstractmethod + def do_splitter_match(self, line: Line) -> TMatchResult: + """ + BaseStringSplitter asks its clients to override this method instead of + `StringTransformer.do_match(...)`. + + Follows the same protocol as `StringTransformer.do_match(...)`. + + Refer to `help(StringTransformer.do_match)` for more information. + """ + + def do_match(self, line: Line) -> TMatchResult: + match_result = self.do_splitter_match(line) + if isinstance(match_result, Err): + return match_result + + string_indices = match_result.ok() + assert len(string_indices) == 1, ( + f"{self.__class__.__name__} should only find one match at a time, found" + f" {len(string_indices)}" + ) + string_idx = string_indices[0] + vresult = self._validate(line, string_idx) + if isinstance(vresult, Err): + return vresult + + return match_result + + def _validate(self, line: Line, string_idx: int) -> TResult[None]: + """ + Checks that @line meets all of the requirements listed in this classes' + docstring. Refer to `help(BaseStringSplitter)` for a detailed + description of those requirements. + + Returns: + * Ok(None), if ALL of the requirements are met. + OR + * Err(CannotTransform), if ANY of the requirements are NOT met. + """ + LL = line.leaves + + string_leaf = LL[string_idx] + + max_string_length = self._get_max_string_length(line, string_idx) + if len(string_leaf.value) <= max_string_length: + return TErr( + "The string itself is not what is causing this line to be too long." + ) + + if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [ + token.STRING, + token.NEWLINE, + ]: + return TErr( + f"This string ({string_leaf.value}) appears to be pointless (i.e. has" + " no parent)." + ) + + if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( + line.comments[id(line.leaves[string_idx])] + ): + return TErr( + "Line appears to end with an inline pragma comment. Splitting the line" + " could modify the pragma's behavior." + ) + + if has_triple_quotes(string_leaf.value): + return TErr("We cannot split multiline strings.") + + return Ok(None) + + def _get_max_string_length(self, line: Line, string_idx: int) -> int: + """ + Calculates the max string length used when attempting to determine + whether or not the target string is responsible for causing the line to + go over the line length limit. + + WARNING: This method is tightly coupled to both StringSplitter and + (especially) StringParenWrapper. There is probably a better way to + accomplish what is being done here. + + Returns: + max_string_length: such that `line.leaves[string_idx].value > + max_string_length` implies that the target string IS responsible + for causing this line to exceed the line length limit. + """ + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + # We use the shorthand "WMA4" in comments to abbreviate "We must + # account for". When giving examples, we use STRING to mean some/any + # valid string. + # + # Finally, we use the following convenience variables: + # + # P: The leaf that is before the target string leaf. + # N: The leaf that is after the target string leaf. + # NN: The leaf that is after N. + + # WMA4 the whitespace at the beginning of the line. + offset = line.depth * 4 + + if is_valid_index(string_idx - 1): + p_idx = string_idx - 1 + if ( + LL[string_idx - 1].type == token.LPAR + and LL[string_idx - 1].value == "" + and string_idx >= 2 + ): + # If the previous leaf is an empty LPAR placeholder, we should skip it. + p_idx -= 1 + + P = LL[p_idx] + if P.type in self.STRING_OPERATORS: + # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`). + offset += len(str(P)) + 1 + + if P.type == token.COMMA: + # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`]. + offset += 3 + + if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]: + # This conditional branch is meant to handle dictionary keys, + # variable assignments, 'return STRING' statement lines, and + # 'else STRING' ternary expression lines. + + # WMA4 a single space. + offset += 1 + + # WMA4 the lengths of any leaves that came before that space, + # but after any closing bracket before that space. + for leaf in reversed(LL[: p_idx + 1]): + offset += len(str(leaf)) + if leaf.type in CLOSING_BRACKETS: + break + + if is_valid_index(string_idx + 1): + N = LL[string_idx + 1] + if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2: + # If the next leaf is an empty RPAR placeholder, we should skip it. + N = LL[string_idx + 2] + + if N.type == token.COMMA: + # WMA4 a single comma at the end of the string (e.g `STRING,`). + offset += 1 + + if is_valid_index(string_idx + 2): + NN = LL[string_idx + 2] + + if N.type == token.DOT and NN.type == token.NAME: + # This conditional branch is meant to handle method calls invoked + # off of a string literal up to and including the LPAR character. + + # WMA4 the '.' character. + offset += 1 + + if ( + is_valid_index(string_idx + 3) + and LL[string_idx + 3].type == token.LPAR + ): + # WMA4 the left parenthesis character. + offset += 1 + + # WMA4 the length of the method's name. + offset += len(NN.value) + + has_comments = False + for comment_leaf in line.comments_after(LL[string_idx]): + if not has_comments: + has_comments = True + # WMA4 two spaces before the '#' character. + offset += 2 + + # WMA4 the length of the inline comment. + offset += len(comment_leaf.value) + + max_string_length = count_chars_in_width(str(line), self.line_length - offset) + return max_string_length + + @staticmethod + def _prefer_paren_wrap_match(LL: list[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the "prefer paren wrap" statement + requirements listed in the 'Requirements' section of the StringParenWrapper + class's docstring. + OR + None, otherwise. + """ + # The line must start with a string. + if LL[0].type != token.STRING: + return None + + matching_nodes = [ + syms.listmaker, + syms.dictsetmaker, + syms.testlist_gexp, + ] + # If the string is an immediate child of a list/set/tuple literal... + if ( + parent_type(LL[0]) in matching_nodes + or parent_type(LL[0].parent) in matching_nodes + ): + # And the string is surrounded by commas (or is the first/last child)... + prev_sibling = LL[0].prev_sibling + next_sibling = LL[0].next_sibling + if ( + not prev_sibling + and not next_sibling + and parent_type(LL[0]) == syms.atom + ): + # If it's an atom string, we need to check the parent atom's siblings. + parent = LL[0].parent + assert parent is not None # For type checkers. + prev_sibling = parent.prev_sibling + next_sibling = parent.next_sibling + if (not prev_sibling or prev_sibling.type == token.COMMA) and ( + not next_sibling or next_sibling.type == token.COMMA + ): + return 0 + + return None + + +def iter_fexpr_spans(s: str) -> Iterator[tuple[int, int]]: + """ + Yields spans corresponding to expressions in a given f-string. + Spans are half-open ranges (left inclusive, right exclusive). + Assumes the input string is a valid f-string, but will not crash if the input + string is invalid. + """ + stack: list[int] = [] # our curly paren stack + i = 0 + while i < len(s): + if s[i] == "{": + # if we're in a string part of the f-string, ignore escaped curly braces + if not stack and i + 1 < len(s) and s[i + 1] == "{": + i += 2 + continue + stack.append(i) + i += 1 + continue + + if s[i] == "}": + if not stack: + i += 1 + continue + j = stack.pop() + # we've made it back out of the expression! yield the span + if not stack: + yield (j, i + 1) + i += 1 + continue + + # if we're in an expression part of the f-string, fast-forward through strings + # note that backslashes are not legal in the expression portion of f-strings + if stack: + delim = None + if s[i : i + 3] in ("'''", '"""'): + delim = s[i : i + 3] + elif s[i] in ("'", '"'): + delim = s[i] + if delim: + i += len(delim) + while i < len(s) and s[i : i + len(delim)] != delim: + i += 1 + i += len(delim) + continue + i += 1 + + +def fstring_contains_expr(s: str) -> bool: + return any(iter_fexpr_spans(s)) + + +def _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str: + """ + Toggles quotes used in f-string expressions that are `old_quote`. + + f-string expressions can't contain backslashes, so we need to toggle the + quotes if the f-string itself will end up using the same quote. We can + simply toggle without escaping because, quotes can't be reused in f-string + expressions. They will fail to parse. + + NOTE: If PEP 701 is accepted, above statement will no longer be true. + Though if quotes can be reused, we can simply reuse them without updates or + escaping, once Black figures out how to parse the new grammar. + """ + new_quote = "'" if old_quote == '"' else '"' + parts = [] + previous_index = 0 + for start, end in iter_fexpr_spans(fstring): + parts.append(fstring[previous_index:start]) + parts.append(fstring[start:end].replace(old_quote, new_quote)) + previous_index = end + parts.append(fstring[previous_index:]) + return "".join(parts) + + +class StringSplitter(BaseStringSplitter, CustomSplitMapMixin): + """ + StringTransformer that splits "atom" strings (i.e. strings which exist on + lines by themselves). + + Requirements: + * The line consists ONLY of a single string (possibly prefixed by a + string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE + a trailing comma. + AND + * All of the requirements listed in BaseStringSplitter's docstring. + + Transformations: + The string mentioned in the 'Requirements' section is split into as + many substrings as necessary to adhere to the configured line length. + + In the final set of substrings, no substring should be smaller than + MIN_SUBSTR_SIZE characters. + + The string will ONLY be split on spaces (i.e. each new substring should + start with a space). Note that the string will NOT be split on a space + which is escaped with a backslash. + + If the string is an f-string, it will NOT be split in the middle of an + f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x + else bar()} is an f-expression). + + If the string that is being split has an associated set of custom split + records and those custom splits will NOT result in any line going over + the configured line length, those custom splits are used. Otherwise the + string is split as late as possible (from left-to-right) while still + adhering to the transformation rules listed above. + + Collaborations: + StringSplitter relies on StringMerger to construct the appropriate + CustomSplit objects and add them to the custom split map. + """ + + MIN_SUBSTR_SIZE: Final = 6 + + def do_splitter_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + if self._prefer_paren_wrap_match(LL) is not None: + return TErr("Line needs to be wrapped in parens first.") + + is_valid_index = is_valid_index_factory(LL) + + idx = 0 + + # The first two leaves MAY be the 'not in' keywords... + if ( + is_valid_index(idx) + and is_valid_index(idx + 1) + and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME] + and str(LL[idx]) + str(LL[idx + 1]) == "not in" + ): + idx += 2 + # Else the first leaf MAY be a string operator symbol or the 'in' keyword... + elif is_valid_index(idx) and ( + LL[idx].type in self.STRING_OPERATORS + or LL[idx].type == token.NAME + and str(LL[idx]) == "in" + ): + idx += 1 + + # The next/first leaf MAY be an empty LPAR... + if is_valid_index(idx) and is_empty_lpar(LL[idx]): + idx += 1 + + # The next/first leaf MUST be a string... + if not is_valid_index(idx) or LL[idx].type != token.STRING: + return TErr("Line does not start with a string.") + + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # That string MAY be followed by an empty RPAR... + if is_valid_index(idx) and is_empty_rpar(LL[idx]): + idx += 1 + + # That string / empty RPAR leaf MAY be followed by a comma... + if is_valid_index(idx) and LL[idx].type == token.COMMA: + idx += 1 + + # But no more leaves are allowed... + if is_valid_index(idx): + return TErr("This line does not end with a string.") + + return Ok([string_idx]) + + def do_transform( + self, line: Line, string_indices: list[int] + ) -> Iterator[TResult[Line]]: + LL = line.leaves + assert len(string_indices) == 1, ( + f"{self.__class__.__name__} should only find one match at a time, found" + f" {len(string_indices)}" + ) + string_idx = string_indices[0] + + QUOTE = LL[string_idx].value[-1] + + is_valid_index = is_valid_index_factory(LL) + insert_str_child = insert_str_child_factory(LL[string_idx]) + + prefix = get_string_prefix(LL[string_idx].value).lower() + + # We MAY choose to drop the 'f' prefix from substrings that don't + # contain any f-expressions, but ONLY if the original f-string + # contains at least one f-expression. Otherwise, we will alter the AST + # of the program. + drop_pointless_f_prefix = ("f" in prefix) and fstring_contains_expr( + LL[string_idx].value + ) + + first_string_line = True + + string_op_leaves = self._get_string_operator_leaves(LL) + string_op_leaves_length = ( + sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1 + if string_op_leaves + else 0 + ) + + def maybe_append_string_operators(new_line: Line) -> None: + """ + Side Effects: + If @line starts with a string operator and this is the first + line we are constructing, this function appends the string + operator to @new_line and replaces the old string operator leaf + in the node structure. Otherwise this function does nothing. + """ + maybe_prefix_leaves = string_op_leaves if first_string_line else [] + for i, prefix_leaf in enumerate(maybe_prefix_leaves): + replace_child(LL[i], prefix_leaf) + new_line.append(prefix_leaf) + + ends_with_comma = ( + is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA + ) + + def max_last_string_column() -> int: + """ + Returns: + The max allowed width of the string value used for the last + line we will construct. Note that this value means the width + rather than the number of characters (e.g., many East Asian + characters expand to two columns). + """ + result = self.line_length + result -= line.depth * 4 + result -= 1 if ends_with_comma else 0 + result -= string_op_leaves_length + return result + + # --- Calculate Max Break Width (for string value) + # We start with the line length limit + max_break_width = self.line_length + # The last index of a string of length N is N-1. + max_break_width -= 1 + # Leading whitespace is not present in the string value (e.g. Leaf.value). + max_break_width -= line.depth * 4 + if max_break_width < 0: + yield TErr( + f"Unable to split {LL[string_idx].value} at such high of a line depth:" + f" {line.depth}" + ) + return + + # Check if StringMerger registered any custom splits. + custom_splits = self.pop_custom_splits(LL[string_idx].value) + # We use them ONLY if none of them would produce lines that exceed the + # line limit. + use_custom_breakpoints = bool( + custom_splits + and all(csplit.break_idx <= max_break_width for csplit in custom_splits) + ) + + # Temporary storage for the remaining chunk of the string line that + # can't fit onto the line currently being constructed. + rest_value = LL[string_idx].value + + def more_splits_should_be_made() -> bool: + """ + Returns: + True iff `rest_value` (the remaining string value from the last + split), should be split again. + """ + if use_custom_breakpoints: + return len(custom_splits) > 1 + else: + return str_width(rest_value) > max_last_string_column() + + string_line_results: list[Ok[Line]] = [] + while more_splits_should_be_made(): + if use_custom_breakpoints: + # Custom User Split (manual) + csplit = custom_splits.pop(0) + break_idx = csplit.break_idx + else: + # Algorithmic Split (automatic) + max_bidx = ( + count_chars_in_width(rest_value, max_break_width) + - string_op_leaves_length + ) + maybe_break_idx = self._get_break_idx(rest_value, max_bidx) + if maybe_break_idx is None: + # If we are unable to algorithmically determine a good split + # and this string has custom splits registered to it, we + # fall back to using them--which means we have to start + # over from the beginning. + if custom_splits: + rest_value = LL[string_idx].value + string_line_results = [] + first_string_line = True + use_custom_breakpoints = True + continue + + # Otherwise, we stop splitting here. + break + + break_idx = maybe_break_idx + + # --- Construct `next_value` + next_value = rest_value[:break_idx] + QUOTE + + # HACK: The following 'if' statement is a hack to fix the custom + # breakpoint index in the case of either: (a) substrings that were + # f-strings but will have the 'f' prefix removed OR (b) substrings + # that were not f-strings but will now become f-strings because of + # redundant use of the 'f' prefix (i.e. none of the substrings + # contain f-expressions but one or more of them had the 'f' prefix + # anyway; in which case, we will prepend 'f' to _all_ substrings). + # + # There is probably a better way to accomplish what is being done + # here... + # + # If this substring is an f-string, we _could_ remove the 'f' + # prefix, and the current custom split did NOT originally use a + # prefix... + if ( + use_custom_breakpoints + and not csplit.has_prefix + and ( + # `next_value == prefix + QUOTE` happens when the custom + # split is an empty string. + next_value == prefix + QUOTE + or next_value != self._normalize_f_string(next_value, prefix) + ) + ): + # Then `csplit.break_idx` will be off by one after removing + # the 'f' prefix. + break_idx += 1 + next_value = rest_value[:break_idx] + QUOTE + + if drop_pointless_f_prefix: + next_value = self._normalize_f_string(next_value, prefix) + + # --- Construct `next_leaf` + next_leaf = Leaf(token.STRING, next_value) + insert_str_child(next_leaf) + self._maybe_normalize_string_quotes(next_leaf) + + # --- Construct `next_line` + next_line = line.clone() + maybe_append_string_operators(next_line) + next_line.append(next_leaf) + string_line_results.append(Ok(next_line)) + + rest_value = prefix + QUOTE + rest_value[break_idx:] + first_string_line = False + + yield from string_line_results + + if drop_pointless_f_prefix: + rest_value = self._normalize_f_string(rest_value, prefix) + + rest_leaf = Leaf(token.STRING, rest_value) + insert_str_child(rest_leaf) + + # NOTE: I could not find a test case that verifies that the following + # line is actually necessary, but it seems to be. Otherwise we risk + # not normalizing the last substring, right? + self._maybe_normalize_string_quotes(rest_leaf) + + last_line = line.clone() + maybe_append_string_operators(last_line) + + # If there are any leaves to the right of the target string... + if is_valid_index(string_idx + 1): + # We use `temp_value` here to determine how long the last line + # would be if we were to append all the leaves to the right of the + # target string to the last string line. + temp_value = rest_value + for leaf in LL[string_idx + 1 :]: + temp_value += str(leaf) + if leaf.type == token.LPAR: + break + + # Try to fit them all on the same line with the last substring... + if ( + str_width(temp_value) <= max_last_string_column() + or LL[string_idx + 1].type == token.COMMA + ): + last_line.append(rest_leaf) + append_leaves(last_line, line, LL[string_idx + 1 :]) + yield Ok(last_line) + # Otherwise, place the last substring on one line and everything + # else on a line below that... + else: + last_line.append(rest_leaf) + yield Ok(last_line) + + non_string_line = line.clone() + append_leaves(non_string_line, line, LL[string_idx + 1 :]) + yield Ok(non_string_line) + # Else the target string was the last leaf... + else: + last_line.append(rest_leaf) + last_line.comments = line.comments.copy() + yield Ok(last_line) + + def _iter_nameescape_slices(self, string: str) -> Iterator[tuple[Index, Index]]: + """ + Yields: + All ranges of @string which, if @string were to be split there, + would result in the splitting of an \\N{...} expression (which is NOT + allowed). + """ + # True - the previous backslash was unescaped + # False - the previous backslash was escaped *or* there was no backslash + previous_was_unescaped_backslash = False + it = iter(enumerate(string)) + for idx, c in it: + if c == "\\": + previous_was_unescaped_backslash = not previous_was_unescaped_backslash + continue + if not previous_was_unescaped_backslash or c != "N": + previous_was_unescaped_backslash = False + continue + previous_was_unescaped_backslash = False + + begin = idx - 1 # the position of backslash before \N{...} + for idx, c in it: + if c == "}": + end = idx + break + else: + # malformed nameescape expression? + # should have been detected by AST parsing earlier... + raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!") + yield begin, end + + def _iter_fexpr_slices(self, string: str) -> Iterator[tuple[Index, Index]]: + """ + Yields: + All ranges of @string which, if @string were to be split there, + would result in the splitting of an f-expression (which is NOT + allowed). + """ + if "f" not in get_string_prefix(string).lower(): + return + yield from iter_fexpr_spans(string) + + def _get_illegal_split_indices(self, string: str) -> set[Index]: + illegal_indices: set[Index] = set() + iterators = [ + self._iter_fexpr_slices(string), + self._iter_nameescape_slices(string), + ] + for it in iterators: + for begin, end in it: + illegal_indices.update(range(begin, end + 1)) + return illegal_indices + + def _get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]: + """ + This method contains the algorithm that StringSplitter uses to + determine which character to split each string at. + + Args: + @string: The substring that we are attempting to split. + @max_break_idx: The ideal break index. We will return this value if it + meets all the necessary conditions. In the likely event that it + doesn't we will try to find the closest index BELOW @max_break_idx + that does. If that fails, we will expand our search by also + considering all valid indices ABOVE @max_break_idx. + + Pre-Conditions: + * assert_is_leaf_string(@string) + * 0 <= @max_break_idx < len(@string) + + Returns: + break_idx, if an index is able to be found that meets all of the + conditions listed in the 'Transformations' section of this classes' + docstring. + OR + None, otherwise. + """ + is_valid_index = is_valid_index_factory(string) + + assert is_valid_index(max_break_idx) + assert_is_leaf_string(string) + + _illegal_split_indices = self._get_illegal_split_indices(string) + + def breaks_unsplittable_expression(i: Index) -> bool: + """ + Returns: + True iff returning @i would result in the splitting of an + unsplittable expression (which is NOT allowed). + """ + return i in _illegal_split_indices + + def passes_all_checks(i: Index) -> bool: + """ + Returns: + True iff ALL of the conditions listed in the 'Transformations' + section of this classes' docstring would be met by returning @i. + """ + is_space = string[i] == " " + is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS + + is_not_escaped = True + j = i - 1 + while is_valid_index(j) and string[j] == "\\": + is_not_escaped = not is_not_escaped + j -= 1 + + is_big_enough = ( + len(string[i:]) >= self.MIN_SUBSTR_SIZE + and len(string[:i]) >= self.MIN_SUBSTR_SIZE + ) + return ( + (is_space or is_split_safe) + and is_not_escaped + and is_big_enough + and not breaks_unsplittable_expression(i) + ) + + # First, we check all indices BELOW @max_break_idx. + break_idx = max_break_idx + while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx): + break_idx -= 1 + + if not passes_all_checks(break_idx): + # If that fails, we check all indices ABOVE @max_break_idx. + # + # If we are able to find a valid index here, the next line is going + # to be longer than the specified line length, but it's probably + # better than doing nothing at all. + break_idx = max_break_idx + 1 + while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx): + break_idx += 1 + + if not is_valid_index(break_idx) or not passes_all_checks(break_idx): + return None + + return break_idx + + def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None: + if self.normalize_strings: + leaf.value = normalize_string_quotes(leaf.value) + + def _normalize_f_string(self, string: str, prefix: str) -> str: + """ + Pre-Conditions: + * assert_is_leaf_string(@string) + + Returns: + * If @string is an f-string that contains no f-expressions, we + return a string identical to @string except that the 'f' prefix + has been stripped and all double braces (i.e. '{{' or '}}') have + been normalized (i.e. turned into '{' or '}'). + OR + * Otherwise, we return @string. + """ + assert_is_leaf_string(string) + + if "f" in prefix and not fstring_contains_expr(string): + new_prefix = prefix.replace("f", "") + + temp = string[len(prefix) :] + temp = re.sub(r"\{\{", "{", temp) + temp = re.sub(r"\}\}", "}", temp) + new_string = temp + + return f"{new_prefix}{new_string}" + else: + return string + + def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> list[Leaf]: + LL = list(leaves) + + string_op_leaves = [] + i = 0 + while LL[i].type in self.STRING_OPERATORS + [token.NAME]: + prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip()) + string_op_leaves.append(prefix_leaf) + i += 1 + return string_op_leaves + + +class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin): + """ + StringTransformer that wraps strings in parens and then splits at the LPAR. + + Requirements: + All of the requirements listed in BaseStringSplitter's docstring in + addition to the requirements listed below: + + * The line is a return/yield statement, which returns/yields a string. + OR + * The line is part of a ternary expression (e.g. `x = y if cond else + z`) such that the line starts with `else <string>`, where <string> is + some string. + OR + * The line is an assert statement, which ends with a string. + OR + * The line is an assignment statement (e.g. `x = <string>` or `x += + <string>`) such that the variable is being assigned the value of some + string. + OR + * The line is a dictionary key assignment where some valid key is being + assigned the value of some string. + OR + * The line is an lambda expression and the value is a string. + OR + * The line starts with an "atom" string that prefers to be wrapped in + parens. It's preferred to be wrapped when it's is an immediate child of + a list/set/tuple literal, AND the string is surrounded by commas (or is + the first/last child). + + Transformations: + The chosen string is wrapped in parentheses and then split at the LPAR. + + We then have one line which ends with an LPAR and another line that + starts with the chosen string. The latter line is then split again at + the RPAR. This results in the RPAR (and possibly a trailing comma) + being placed on its own line. + + NOTE: If any leaves exist to the right of the chosen string (except + for a trailing comma, which would be placed after the RPAR), those + leaves are placed inside the parentheses. In effect, the chosen + string is not necessarily being "wrapped" by parentheses. We can, + however, count on the LPAR being placed directly before the chosen + string. + + In other words, StringParenWrapper creates "atom" strings. These + can then be split again by StringSplitter, if necessary. + + Collaborations: + In the event that a string line split by StringParenWrapper is + changed such that it no longer needs to be given its own line, + StringParenWrapper relies on StringParenStripper to clean up the + parentheses it created. + + For "atom" strings that prefers to be wrapped in parens, it requires + StringSplitter to hold the split until the string is wrapped in parens. + """ + + def do_splitter_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + if line.leaves[-1].type in OPENING_BRACKETS: + return TErr( + "Cannot wrap parens around a line that ends in an opening bracket." + ) + + string_idx = ( + self._return_match(LL) + or self._else_match(LL) + or self._assert_match(LL) + or self._assign_match(LL) + or self._dict_or_lambda_match(LL) + or self._prefer_paren_wrap_match(LL) + ) + + if string_idx is not None: + string_value = line.leaves[string_idx].value + # If the string has neither spaces nor East Asian stops... + if not any( + char == " " or char in SPLIT_SAFE_CHARS for char in string_value + ): + # And will still violate the line length limit when split... + max_string_width = self.line_length - ((line.depth + 1) * 4) + if str_width(string_value) > max_string_width: + # And has no associated custom splits... + if not self.has_custom_splits(string_value): + # Then we should NOT put this string on its own line. + return TErr( + "We do not wrap long strings in parentheses when the" + " resultant line would still be over the specified line" + " length and can't be split further by StringSplitter." + ) + return Ok([string_idx]) + + return TErr("This line does not contain any non-atomic strings.") + + @staticmethod + def _return_match(LL: list[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the return/yield statement + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is a part of a return/yield statement and the first leaf + # contains either the "return" or "yield" keywords... + if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[ + 0 + ].value in ["return", "yield"]: + is_valid_index = is_valid_index_factory(LL) + + idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1 + # The next visible leaf MUST contain a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + return idx + + return None + + @staticmethod + def _else_match(LL: list[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the ternary expression + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is a part of a ternary expression and the first leaf + # contains the "else" keyword... + if ( + parent_type(LL[0]) == syms.test + and LL[0].type == token.NAME + and LL[0].value == "else" + ): + is_valid_index = is_valid_index_factory(LL) + + idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1 + # The next visible leaf MUST contain a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + return idx + + return None + + @staticmethod + def _assert_match(LL: list[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the assert statement + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is a part of an assert statement and the first leaf + # contains the "assert" keyword... + if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert": + is_valid_index = is_valid_index_factory(LL) + + for i, leaf in enumerate(LL): + # We MUST find a comma... + if leaf.type == token.COMMA: + idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 + + # That comma MUST be followed by a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # But no more leaves are allowed... + if not is_valid_index(idx): + return string_idx + + return None + + @staticmethod + def _assign_match(LL: list[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the assignment statement + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is a part of an expression statement or is a function + # argument AND the first leaf contains a variable name... + if ( + parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power] + and LL[0].type == token.NAME + ): + is_valid_index = is_valid_index_factory(LL) + + for i, leaf in enumerate(LL): + # We MUST find either an '=' or '+=' symbol... + if leaf.type in [token.EQUAL, token.PLUSEQUAL]: + idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 + + # That symbol MUST be followed by a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # The next leaf MAY be a comma iff this line is a part + # of a function argument... + if ( + parent_type(LL[0]) == syms.argument + and is_valid_index(idx) + and LL[idx].type == token.COMMA + ): + idx += 1 + + # But no more leaves are allowed... + if not is_valid_index(idx): + return string_idx + + return None + + @staticmethod + def _dict_or_lambda_match(LL: list[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the dictionary key assignment + statement or lambda expression requirements listed in the + 'Requirements' section of this classes' docstring. + OR + None, otherwise. + """ + # If this line is a part of a dictionary key assignment or lambda expression... + parent_types = [parent_type(LL[0]), parent_type(LL[0].parent)] + if syms.dictsetmaker in parent_types or syms.lambdef in parent_types: + is_valid_index = is_valid_index_factory(LL) + + for i, leaf in enumerate(LL): + # We MUST find a colon, it can either be dict's or lambda's colon... + if leaf.type == token.COLON and i < len(LL) - 1: + idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 + + # That colon MUST be followed by a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # That string MAY be followed by a comma... + if is_valid_index(idx) and LL[idx].type == token.COMMA: + idx += 1 + + # But no more leaves are allowed... + if not is_valid_index(idx): + return string_idx + + return None + + def do_transform( + self, line: Line, string_indices: list[int] + ) -> Iterator[TResult[Line]]: + LL = line.leaves + assert len(string_indices) == 1, ( + f"{self.__class__.__name__} should only find one match at a time, found" + f" {len(string_indices)}" + ) + string_idx = string_indices[0] + + is_valid_index = is_valid_index_factory(LL) + insert_str_child = insert_str_child_factory(LL[string_idx]) + + comma_idx = -1 + ends_with_comma = False + if LL[comma_idx].type == token.COMMA: + ends_with_comma = True + + leaves_to_steal_comments_from = [LL[string_idx]] + if ends_with_comma: + leaves_to_steal_comments_from.append(LL[comma_idx]) + + # --- First Line + first_line = line.clone() + left_leaves = LL[:string_idx] + + # We have to remember to account for (possibly invisible) LPAR and RPAR + # leaves that already wrapped the target string. If these leaves do + # exist, we will replace them with our own LPAR and RPAR leaves. + old_parens_exist = False + if left_leaves and left_leaves[-1].type == token.LPAR: + old_parens_exist = True + leaves_to_steal_comments_from.append(left_leaves[-1]) + left_leaves.pop() + + append_leaves(first_line, line, left_leaves) + + lpar_leaf = Leaf(token.LPAR, "(") + if old_parens_exist: + replace_child(LL[string_idx - 1], lpar_leaf) + else: + insert_str_child(lpar_leaf) + first_line.append(lpar_leaf) + + # We throw inline comments that were originally to the right of the + # target string to the top line. They will now be shown to the right of + # the LPAR. + for leaf in leaves_to_steal_comments_from: + for comment_leaf in line.comments_after(leaf): + first_line.append(comment_leaf, preformatted=True) + + yield Ok(first_line) + + # --- Middle (String) Line + # We only need to yield one (possibly too long) string line, since the + # `StringSplitter` will break it down further if necessary. + string_value = LL[string_idx].value + string_line = Line( + mode=line.mode, + depth=line.depth + 1, + inside_brackets=True, + should_split_rhs=line.should_split_rhs, + magic_trailing_comma=line.magic_trailing_comma, + ) + string_leaf = Leaf(token.STRING, string_value) + insert_str_child(string_leaf) + string_line.append(string_leaf) + + old_rpar_leaf = None + if is_valid_index(string_idx + 1): + right_leaves = LL[string_idx + 1 :] + if ends_with_comma: + right_leaves.pop() + + if old_parens_exist: + assert right_leaves and right_leaves[-1].type == token.RPAR, ( + "Apparently, old parentheses do NOT exist?!" + f" (left_leaves={left_leaves}, right_leaves={right_leaves})" + ) + old_rpar_leaf = right_leaves.pop() + elif right_leaves and right_leaves[-1].type == token.RPAR: + # Special case for lambda expressions as dict's value, e.g.: + # my_dict = { + # "key": lambda x: f"formatted: {x}", + # } + # After wrapping the dict's value with parentheses, the string is + # followed by a RPAR but its opening bracket is lambda's, not + # the string's: + # "key": (lambda x: f"formatted: {x}"), + opening_bracket = right_leaves[-1].opening_bracket + if opening_bracket is not None and opening_bracket in left_leaves: + index = left_leaves.index(opening_bracket) + if ( + 0 < index < len(left_leaves) - 1 + and left_leaves[index - 1].type == token.COLON + and left_leaves[index + 1].value == "lambda" + ): + right_leaves.pop() + + append_leaves(string_line, line, right_leaves) + + yield Ok(string_line) + + # --- Last Line + last_line = line.clone() + last_line.bracket_tracker = first_line.bracket_tracker + + new_rpar_leaf = Leaf(token.RPAR, ")") + if old_rpar_leaf is not None: + replace_child(old_rpar_leaf, new_rpar_leaf) + else: + insert_str_child(new_rpar_leaf) + last_line.append(new_rpar_leaf) + + # If the target string ended with a comma, we place this comma to the + # right of the RPAR on the last line. + if ends_with_comma: + comma_leaf = Leaf(token.COMMA, ",") + replace_child(LL[comma_idx], comma_leaf) + last_line.append(comma_leaf) + + yield Ok(last_line) + + +class StringParser: + """ + A state machine that aids in parsing a string's "trailer", which can be + either non-existent, an old-style formatting sequence (e.g. `% varX` or `% + (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX, + varY)`). + + NOTE: A new StringParser object MUST be instantiated for each string + trailer we need to parse. + + Examples: + We shall assume that `line` equals the `Line` object that corresponds + to the following line of python code: + ``` + x = "Some {}.".format("String") + some_other_string + ``` + + Furthermore, we will assume that `string_idx` is some index such that: + ``` + assert line.leaves[string_idx].value == "Some {}." + ``` + + The following code snippet then holds: + ``` + string_parser = StringParser() + idx = string_parser.parse(line.leaves, string_idx) + assert line.leaves[idx].type == token.PLUS + ``` + """ + + DEFAULT_TOKEN: Final = 20210605 + + # String Parser States + START: Final = 1 + DOT: Final = 2 + NAME: Final = 3 + PERCENT: Final = 4 + SINGLE_FMT_ARG: Final = 5 + LPAR: Final = 6 + RPAR: Final = 7 + DONE: Final = 8 + + # Lookup Table for Next State + _goto: Final[dict[tuple[ParserState, NodeType], ParserState]] = { + # A string trailer may start with '.' OR '%'. + (START, token.DOT): DOT, + (START, token.PERCENT): PERCENT, + (START, DEFAULT_TOKEN): DONE, + # A '.' MUST be followed by an attribute or method name. + (DOT, token.NAME): NAME, + # A method name MUST be followed by an '(', whereas an attribute name + # is the last symbol in the string trailer. + (NAME, token.LPAR): LPAR, + (NAME, DEFAULT_TOKEN): DONE, + # A '%' symbol can be followed by an '(' or a single argument (e.g. a + # string or variable name). + (PERCENT, token.LPAR): LPAR, + (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG, + # If a '%' symbol is followed by a single argument, that argument is + # the last leaf in the string trailer. + (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE, + # If present, a ')' symbol is the last symbol in a string trailer. + # (NOTE: LPARS and nested RPARS are not included in this lookup table, + # since they are treated as a special case by the parsing logic in this + # classes' implementation.) + (RPAR, DEFAULT_TOKEN): DONE, + } + + def __init__(self) -> None: + self._state = self.START + self._unmatched_lpars = 0 + + def parse(self, leaves: list[Leaf], string_idx: int) -> int: + """ + Pre-conditions: + * @leaves[@string_idx].type == token.STRING + + Returns: + The index directly after the last leaf which is a part of the string + trailer, if a "trailer" exists. + OR + @string_idx + 1, if no string "trailer" exists. + """ + assert leaves[string_idx].type == token.STRING + + idx = string_idx + 1 + while idx < len(leaves) and self._next_state(leaves[idx]): + idx += 1 + return idx + + def _next_state(self, leaf: Leaf) -> bool: + """ + Pre-conditions: + * On the first call to this function, @leaf MUST be the leaf that + was directly after the string leaf in question (e.g. if our target + string is `line.leaves[i]` then the first call to this method must + be `line.leaves[i + 1]`). + * On the next call to this function, the leaf parameter passed in + MUST be the leaf directly following @leaf. + + Returns: + True iff @leaf is a part of the string's trailer. + """ + # We ignore empty LPAR or RPAR leaves. + if is_empty_par(leaf): + return True + + next_token = leaf.type + if next_token == token.LPAR: + self._unmatched_lpars += 1 + + current_state = self._state + + # The LPAR parser state is a special case. We will return True until we + # find the matching RPAR token. + if current_state == self.LPAR: + if next_token == token.RPAR: + self._unmatched_lpars -= 1 + if self._unmatched_lpars == 0: + self._state = self.RPAR + # Otherwise, we use a lookup table to determine the next state. + else: + # If the lookup table matches the current state to the next + # token, we use the lookup table. + if (current_state, next_token) in self._goto: + self._state = self._goto[current_state, next_token] + else: + # Otherwise, we check if a the current state was assigned a + # default. + if (current_state, self.DEFAULT_TOKEN) in self._goto: + self._state = self._goto[current_state, self.DEFAULT_TOKEN] + # If no default has been assigned, then this parser has a logic + # error. + else: + raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!") + + if self._state == self.DONE: + return False + + return True + + +def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]: + """ + Factory for a convenience function that is used to orphan @string_leaf + and then insert multiple new leaves into the same part of the node + structure that @string_leaf had originally occupied. + + Examples: + Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N = + string_leaf.parent`. Assume the node `N` has the following + original structure: + + Node( + expr_stmt, [ + Leaf(NAME, 'x'), + Leaf(EQUAL, '='), + Leaf(STRING, '"foo"'), + ] + ) + + We then run the code snippet shown below. + ``` + insert_str_child = insert_str_child_factory(string_leaf) + + lpar = Leaf(token.LPAR, '(') + insert_str_child(lpar) + + bar = Leaf(token.STRING, '"bar"') + insert_str_child(bar) + + rpar = Leaf(token.RPAR, ')') + insert_str_child(rpar) + ``` + + After which point, it follows that `string_leaf.parent is None` and + the node `N` now has the following structure: + + Node( + expr_stmt, [ + Leaf(NAME, 'x'), + Leaf(EQUAL, '='), + Leaf(LPAR, '('), + Leaf(STRING, '"bar"'), + Leaf(RPAR, ')'), + ] + ) + """ + string_parent = string_leaf.parent + string_child_idx = string_leaf.remove() + + def insert_str_child(child: LN) -> None: + nonlocal string_child_idx + + assert string_parent is not None + assert string_child_idx is not None + + string_parent.insert_child(string_child_idx, child) + string_child_idx += 1 + + return insert_str_child + + +def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]: + """ + Examples: + ``` + my_list = [1, 2, 3] + + is_valid_index = is_valid_index_factory(my_list) + + assert is_valid_index(0) + assert is_valid_index(2) + + assert not is_valid_index(3) + assert not is_valid_index(-1) + ``` + """ + + def is_valid_index(idx: int) -> bool: + """ + Returns: + True iff @idx is positive AND seq[@idx] does NOT raise an + IndexError. + """ + return 0 <= idx < len(seq) + + return is_valid_index diff --git a/contrib/python/black/blackd/__init__.py b/contrib/python/black/blackd/__init__.py new file mode 100644 index 00000000000..86309da0ef0 --- /dev/null +++ b/contrib/python/black/blackd/__init__.py @@ -0,0 +1,262 @@ +import asyncio +import logging +from concurrent.futures import Executor, ProcessPoolExecutor +from datetime import datetime, timezone +from functools import cache, partial +from multiprocessing import freeze_support + +try: + from aiohttp import web + from multidict import MultiMapping + + from .middlewares import cors +except ImportError as ie: + raise ImportError( + f"aiohttp dependency is not installed: {ie}. " + + "Please re-install black with the '[d]' extra install " + + "to obtain aiohttp_cors: `pip install black[d]`" + ) from None + +import click + +import black +from _black_version import version as __version__ +from black.concurrency import maybe_install_uvloop + +# This is used internally by tests to shut down the server prematurely +_stop_signal = asyncio.Event() + +# Request headers +PROTOCOL_VERSION_HEADER = "X-Protocol-Version" +LINE_LENGTH_HEADER = "X-Line-Length" +PYTHON_VARIANT_HEADER = "X-Python-Variant" +SKIP_SOURCE_FIRST_LINE = "X-Skip-Source-First-Line" +SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" +SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" +PREVIEW = "X-Preview" +UNSTABLE = "X-Unstable" +ENABLE_UNSTABLE_FEATURE = "X-Enable-Unstable-Feature" +FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe" +DIFF_HEADER = "X-Diff" + +BLACK_HEADERS = [ + PROTOCOL_VERSION_HEADER, + LINE_LENGTH_HEADER, + PYTHON_VARIANT_HEADER, + SKIP_SOURCE_FIRST_LINE, + SKIP_STRING_NORMALIZATION_HEADER, + SKIP_MAGIC_TRAILING_COMMA, + PREVIEW, + UNSTABLE, + ENABLE_UNSTABLE_FEATURE, + FAST_OR_SAFE_HEADER, + DIFF_HEADER, +] + +# Response headers +BLACK_VERSION_HEADER = "X-Black-Version" + + +class HeaderError(Exception): + pass + + +class InvalidVariantHeader(Exception): + pass + + [email protected](context_settings={"help_option_names": ["-h", "--help"]}) + "--bind-host", + type=str, + help="Address to bind the server to.", + default="localhost", + show_default=True, +) + "--bind-port", type=int, help="Port to listen on", default=45484, show_default=True +) [email protected]_option(version=black.__version__) +def main(bind_host: str, bind_port: int) -> None: + logging.basicConfig(level=logging.INFO) + app = make_app() + ver = black.__version__ + black.out(f"blackd version {ver} listening on {bind_host} port {bind_port}") + web.run_app(app, host=bind_host, port=bind_port, handle_signals=True, print=None) + + +@cache +def executor() -> Executor: + return ProcessPoolExecutor() + + +def make_app() -> web.Application: + app = web.Application( + middlewares=[cors(allow_headers=(*BLACK_HEADERS, "Content-Type"))] + ) + app.add_routes([web.post("/", partial(handle, executor=executor()))]) + return app + + +async def handle(request: web.Request, executor: Executor) -> web.Response: + headers = {BLACK_VERSION_HEADER: __version__} + try: + if request.headers.get(PROTOCOL_VERSION_HEADER, "1") != "1": + return web.Response( + status=501, text="This server only supports protocol version 1" + ) + + fast = False + if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": + fast = True + try: + mode = parse_mode(request.headers) + except HeaderError as e: + return web.Response(status=400, text=e.args[0]) + req_bytes = await request.content.read() + charset = request.charset if request.charset is not None else "utf8" + req_str = req_bytes.decode(charset) + then = datetime.now(timezone.utc) + + header = "" + if mode.skip_source_first_line: + first_newline_position: int = req_str.find("\n") + 1 + header = req_str[:first_newline_position] + req_str = req_str[first_newline_position:] + + loop = asyncio.get_event_loop() + formatted_str = await loop.run_in_executor( + executor, partial(black.format_file_contents, req_str, fast=fast, mode=mode) + ) + + # Preserve CRLF line endings + nl = req_str.find("\n") + if nl > 0 and req_str[nl - 1] == "\r": + formatted_str = formatted_str.replace("\n", "\r\n") + # If, after swapping line endings, nothing changed, then say so + if formatted_str == req_str: + raise black.NothingChanged + + # Put the source first line back + req_str = header + req_str + formatted_str = header + formatted_str + + # Only output the diff in the HTTP response + only_diff = bool(request.headers.get(DIFF_HEADER, False)) + if only_diff: + now = datetime.now(timezone.utc) + src_name = f"In\t{then}" + dst_name = f"Out\t{now}" + loop = asyncio.get_event_loop() + formatted_str = await loop.run_in_executor( + executor, + partial(black.diff, req_str, formatted_str, src_name, dst_name), + ) + + return web.Response( + content_type=request.content_type, + charset=charset, + headers=headers, + text=formatted_str, + ) + except black.NothingChanged: + return web.Response(status=204, headers=headers) + except black.InvalidInput as e: + return web.Response(status=400, headers=headers, text=str(e)) + except Exception as e: + logging.exception("Exception during handling a request") + return web.Response(status=500, headers=headers, text=str(e)) + + +def parse_mode(headers: MultiMapping[str]) -> black.Mode: + try: + line_length = int(headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH)) + except ValueError: + raise HeaderError("Invalid line length header value") from None + + if PYTHON_VARIANT_HEADER in headers: + value = headers[PYTHON_VARIANT_HEADER] + try: + pyi, versions = parse_python_variant_header(value) + except InvalidVariantHeader as e: + raise HeaderError( + f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", + ) from None + else: + pyi = False + versions = set() + + skip_string_normalization = bool( + headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) + ) + skip_magic_trailing_comma = bool(headers.get(SKIP_MAGIC_TRAILING_COMMA, False)) + skip_source_first_line = bool(headers.get(SKIP_SOURCE_FIRST_LINE, False)) + + preview = bool(headers.get(PREVIEW, False)) + unstable = bool(headers.get(UNSTABLE, False)) + enable_features: set[black.Preview] = set() + enable_unstable_features = headers.get(ENABLE_UNSTABLE_FEATURE, "").split(",") + for piece in enable_unstable_features: + piece = piece.strip() + if piece: + try: + enable_features.add(black.Preview[piece]) + except KeyError: + raise HeaderError( + f"Invalid value for {ENABLE_UNSTABLE_FEATURE}: {piece}", + ) from None + + return black.FileMode( + target_versions=versions, + is_pyi=pyi, + line_length=line_length, + skip_source_first_line=skip_source_first_line, + string_normalization=not skip_string_normalization, + magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, + unstable=unstable, + enabled_features=enable_features, + ) + + +def parse_python_variant_header(value: str) -> tuple[bool, set[black.TargetVersion]]: + if value == "pyi": + return True, set() + else: + versions = set() + for version in value.split(","): + if version.startswith("py"): + version = version[len("py") :] + if "." in version: + major_str, *rest = version.split(".") + else: + major_str = version[0] + rest = [version[1:]] if len(version) > 1 else [] + try: + major = int(major_str) + if major not in (2, 3): + raise InvalidVariantHeader("major version must be 2 or 3") + if len(rest) > 0: + minor = int(rest[0]) + if major == 2: + raise InvalidVariantHeader("Python 2 is not supported") + else: + # Default to lowest supported minor version. + minor = 7 if major == 2 else 3 + version_str = f"PY{major}{minor}" + if major == 3 and not hasattr(black.TargetVersion, version_str): + raise InvalidVariantHeader(f"3.{minor} is not supported") + versions.add(black.TargetVersion[version_str]) + except (KeyError, ValueError): + raise InvalidVariantHeader("expected e.g. '3.7', 'py3.5'") from None + return False, versions + + +def patched_main() -> None: + maybe_install_uvloop() + freeze_support() + main() + + +if __name__ == "__main__": + patched_main() diff --git a/contrib/python/black/blackd/__main__.py b/contrib/python/black/blackd/__main__.py new file mode 100644 index 00000000000..b5a4b137446 --- /dev/null +++ b/contrib/python/black/blackd/__main__.py @@ -0,0 +1,3 @@ +import blackd + +blackd.patched_main() diff --git a/contrib/python/black/blackd/middlewares.py b/contrib/python/black/blackd/middlewares.py new file mode 100644 index 00000000000..43448c2514a --- /dev/null +++ b/contrib/python/black/blackd/middlewares.py @@ -0,0 +1,35 @@ +from collections.abc import Awaitable, Callable, Iterable + +from aiohttp.typedefs import Middleware +from aiohttp.web_middlewares import middleware +from aiohttp.web_request import Request +from aiohttp.web_response import StreamResponse + +Handler = Callable[[Request], Awaitable[StreamResponse]] + + +def cors(allow_headers: Iterable[str]) -> Middleware: + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + is_options = request.method == "OPTIONS" + is_preflight = is_options and "Access-Control-Request-Method" in request.headers + if is_preflight: + resp = StreamResponse() + else: + resp = await handler(request) + + origin = request.headers.get("Origin") + if not origin: + return resp + + resp.headers["Access-Control-Allow-Origin"] = "*" + resp.headers["Access-Control-Expose-Headers"] = "*" + if is_options: + resp.headers["Access-Control-Allow-Headers"] = ", ".join(allow_headers) + resp.headers["Access-Control-Allow-Methods"] = ", ".join( + ("OPTIONS", "POST") + ) + + return resp + + return impl diff --git a/contrib/python/black/blib2to3/Grammar.txt b/contrib/python/black/blib2to3/Grammar.txt new file mode 100644 index 00000000000..c8800e21931 --- /dev/null +++ b/contrib/python/black/blib2to3/Grammar.txt @@ -0,0 +1,261 @@ +# Grammar for 2to3. This grammar supports Python 2.x and 3.x. + +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ + +# Start symbols for the grammar: +# file_input is a module or sequence of commands read from an input file; +# single_input is a single interactive statement; +# eval_input is the input for the eval() and input() functions. +# NB: compound_stmt in single_input is followed by extra NEWLINE! +file_input: (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +eval_input: testlist NEWLINE* ENDMARKER + +typevar: NAME [':' expr] ['=' expr] +paramspec: '**' NAME ['=' expr] +typevartuple: '*' NAME ['=' (expr|star_expr)] +typeparam: typevar | paramspec | typevartuple +typeparams: '[' typeparam (',' typeparam)* [','] ']' + +decorator: '@' namedexpr_test NEWLINE +decorators: decorator+ +decorated: decorators (classdef | funcdef | async_funcdef) +async_funcdef: ASYNC funcdef +funcdef: 'def' NAME [typeparams] parameters ['->' test] ':' suite +parameters: '(' [typedargslist] ')' + +# The following definition for typedarglist is equivalent to this set of rules: +# +# arguments = argument (',' argument)* +# argument = tfpdef ['=' test] +# kwargs = '**' tname [','] +# args = '*' [tname_star] +# kwonly_kwargs = (',' argument)* [',' [kwargs]] +# args_kwonly_kwargs = args kwonly_kwargs | kwargs +# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] +# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs +# typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)" +# +# It needs to be fully expanded to allow our LL(1) parser to work on it. + +typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ + ',' [((tfpdef ['=' test] ',')* ('*' [tname_star] (',' tname ['=' test])* + [',' ['**' tname [',']]] | '**' tname [',']) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] + ] | ((tfpdef ['=' test] ',')* ('*' [tname_star] (',' tname ['=' test])* + [',' ['**' tname [',']]] | '**' tname [',']) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + +tname: NAME [':' test] +tname_star: NAME [':' (test|star_expr)] +tfpdef: tname | '(' tfplist ')' +tfplist: tfpdef (',' tfpdef)* [','] + +# The following definition for varargslist is equivalent to this set of rules: +# +# arguments = argument (',' argument )* +# argument = vfpdef ['=' test] +# kwargs = '**' vname [','] +# args = '*' [vname] +# kwonly_kwargs = (',' argument )* [',' [kwargs]] +# args_kwonly_kwargs = args kwonly_kwargs | kwargs +# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] +# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs +# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly) +# +# It needs to be fully expanded to allow our LL(1) parser to work on it. + +varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ + ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])* + [',' ['**' vname [',']]] | '**' vname [',']) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + ]] | ((vfpdef ['=' test] ',')* + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]]| '**' vname [',']) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + +vname: NAME +vfpdef: vname | '(' vfplist ')' +vfplist: vfpdef (',' vfpdef)* [','] + +stmt: simple_stmt | compound_stmt +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +small_stmt: (type_stmt | expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | assert_stmt) +expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +annassign: ':' test ['=' (yield_expr|testlist_star_expr)] +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') +# For normal and annotated assignments, additional restrictions enforced by the interpreter +del_stmt: 'del' exprlist +pass_stmt: 'pass' +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +break_stmt: 'break' +continue_stmt: 'continue' +return_stmt: 'return' [testlist_star_expr] +yield_stmt: yield_expr +raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] +import_stmt: import_name | import_from +import_name: 'import' dotted_as_names +import_from: ('from' ('.'* dotted_name | '.'+) + 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_as_name: NAME ['as' NAME] +dotted_as_name: dotted_name ['as' NAME] +import_as_names: import_as_name (',' import_as_name)* [','] +dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_name: NAME ('.' NAME)* +global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* +assert_stmt: 'assert' test [',' test] +type_stmt: "type" NAME [typeparams] '=' test + +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] +while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist_star_expr ':' suite ['else' ':' suite] +try_stmt: ('try' ':' suite + ((except_clause ':' suite)+ + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' asexpr_test (',' asexpr_test)* ':' suite + +# NB compile.c makes sure that the default except clause is last +except_clause: 'except' ['*'] [test [(',' | 'as') test]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + +# Backward compatibility cruft to support: +# [ x for x in lambda: True, lambda: False if x() ] +# even while also allowing: +# lambda x: 5 if x else 2 +# (But not a mix of the two) +testlist_safe: old_test [(',' old_test)+ [',']] +old_test: or_test | old_lambdef +old_lambdef: 'lambda' [varargslist] ':' old_test + +namedexpr_test: asexpr_test [':=' asexpr_test] + +# This is actually not a real rule, though since the parser is very +# limited in terms of the strategy about match/case rules, we are inserting +# a virtual case (<expr> as <expr>) as a valid expression. Unless a better +# approach is thought, the only side effect of this seem to be just allowing +# more stuff to be parser (which would fail on the ast). +asexpr_test: test ['as' test] + +test: or_test ['if' or_test 'else' test] | lambdef +or_test: and_test ('or' and_test)* +and_test: not_test ('and' not_test)* +not_test: 'not' not_test | comparison +comparison: expr (comp_op expr)* +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr +expr: xor_expr ('|' xor_expr)* +xor_expr: and_expr ('^' and_expr)* +and_expr: shift_expr ('&' shift_expr)* +shift_expr: arith_expr (('<<'|'>>') arith_expr)* +arith_expr: term (('+'|'-') term)* +term: factor (('*'|'@'|'/'|'%'|'//') factor)* +factor: ('+'|'-'|'~') factor | power +power: [AWAIT] atom trailer* ['**' factor] +atom: ('(' [yield_expr|testlist_gexp] ')' | + '[' [listmaker] ']' | + '{' [dictsetmaker] '}' | + '`' testlist1 '`' | + NAME | NUMBER | (STRING | fstring)+ | '.' '.' '.') +listmaker: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) +testlist_gexp: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) +lambdef: 'lambda' [varargslist] ':' test +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +subscriptlist: (subscript|star_expr) (',' (subscript|star_expr))* [','] +subscript: test [':=' test] | [test] ':' [test] [sliceop] +sliceop: ':' [test] +exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +testlist: test (',' test)* [','] +dictsetmaker: ( ((test ':' asexpr_test | '**' expr) + (comp_for | (',' (test ':' asexpr_test | '**' expr))* [','])) | + ((test [':=' test] | star_expr) + (comp_for | (',' (test [':=' test] | star_expr))* [','])) ) + +classdef: 'class' NAME [typeparams] ['(' [arglist] ')'] ':' suite + +arglist: argument (',' argument)* [','] + +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguments are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test ':=' test [comp_for] | + test 'as' test | + test '=' asexpr_test | + '**' test | + '*' test ) + +comp_iter: comp_for | comp_if +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' old_test [comp_iter] + +# As noted above, testlist_safe extends the syntax allowed in list +# comprehensions and generators. We can't use it indiscriminately in all +# derivations using a comp_for-like pattern because the testlist_safe derivation +# contains comma which clashes with trailing comma in arglist. +# +# This was an issue because the parser would not follow the correct derivation +# when parsing syntactically valid Python code. Since testlist_safe was created +# specifically to handle list comprehensions and generator expressions enclosed +# with parentheses, it's safe to only use it in those. That avoids the issue; we +# can parse code like set(x for x in [],). +# +# The syntax supported by this set of rules is not a valid Python 3 syntax, +# hence the prefix "old". +# +# See https://bugs.python.org/issue27494 +old_comp_iter: old_comp_for | old_comp_if +old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_if: 'if' old_test [old_comp_iter] + +testlist1: test (',' test)* + +# not used in grammar, but may appear in "node" passed from Parser to Compiler +encoding_decl: NAME + +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist_star_expr + + +# 3.10 match statement definition + +# PS: normally the grammar is much much more restricted, but +# at this moment for not trying to bother much with encoding the +# exact same DSL in a LL(1) parser, we will just accept an expression +# and let the ast.parse() step of the safe mode to reject invalid +# grammar. + +# The reason why it is more restricted is that, patterns are some +# sort of a DSL (more advanced than our LHS on assignments, but +# still in a very limited python subset). They are not really +# expressions, but who cares. If we can parse them, that is enough +# to reformat them. + +match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT + +# This is more permissive than the actual version. For example it +# accepts `match *something:`, even though single-item starred expressions +# are forbidden. +subject_expr: (namedexpr_test|star_expr) (',' (namedexpr_test|star_expr))* [','] + +# cases +case_block: "case" patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern (',' pattern)* [','] +pattern: (expr|star_expr) ['as' expr] + +fstring: FSTRING_START fstring_middle* FSTRING_END +fstring_middle: fstring_replacement_field | FSTRING_MIDDLE +fstring_replacement_field: '{' (yield_expr | testlist_star_expr) ['='] [ "!" NAME ] [ ':' fstring_format_spec* ] '}' +fstring_format_spec: FSTRING_MIDDLE | fstring_replacement_field diff --git a/contrib/python/black/blib2to3/LICENSE b/contrib/python/black/blib2to3/LICENSE new file mode 100644 index 00000000000..ef8df0698b7 --- /dev/null +++ b/contrib/python/black/blib2to3/LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/contrib/python/black/blib2to3/PatternGrammar.txt b/contrib/python/black/blib2to3/PatternGrammar.txt new file mode 100644 index 00000000000..36bf8148273 --- /dev/null +++ b/contrib/python/black/blib2to3/PatternGrammar.txt @@ -0,0 +1,28 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# A grammar to describe tree matching patterns. +# Not shown here: +# - 'TOKEN' stands for any token (leaf node) +# - 'any' stands for any node (leaf or interior) +# With 'any' we can still specify the sub-structure. + +# The start symbol is 'Matcher'. + +Matcher: Alternatives ENDMARKER + +Alternatives: Alternative ('|' Alternative)* + +Alternative: (Unit | NegatedUnit)+ + +Unit: [NAME '='] ( STRING [Repeater] + | NAME [Details] [Repeater] + | '(' Alternatives ')' [Repeater] + | '[' Alternatives ']' + ) + +NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') + +Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' + +Details: '<' Alternatives '>' diff --git a/contrib/python/black/blib2to3/README b/contrib/python/black/blib2to3/README new file mode 100644 index 00000000000..38b04158ddb --- /dev/null +++ b/contrib/python/black/blib2to3/README @@ -0,0 +1,24 @@ +A subset of lib2to3 taken from Python 3.7.0b2. Commit hash: +9c17e3a1987004b8bcfbe423953aad84493a7984 + +Reasons for forking: + +- consistent handling of f-strings for users of Python < 3.6.2 +- backport of BPO-33064 that fixes parsing files with trailing commas after \*args and + \*\*kwargs +- backport of GH-6143 that restores the ability to reformat legacy usage of `async` +- support all types of string literals +- better ability to debug (better reprs) +- INDENT and DEDENT don't hold whitespace and comment prefixes +- ability to Cythonize + +Change Log: + +- Changes default logger used by Driver +- Backported the following upstream parser changes: + - "bpo-42381: Allow walrus in set literals and set comprehensions (GH-23332)" + https://github.com/python/cpython/commit/cae60187cf7a7b26281d012e1952fafe4e2e97e9 + - "bpo-42316: Allow unparenthesized walrus operator in indexes (GH-23317)" + https://github.com/python/cpython/commit/b0aba1fcdc3da952698d99aec2334faa79a8b68c +- Tweaks to help mypyc compile faster code (including inlining type information, + "Final-ing", etc.) diff --git a/contrib/python/black/blib2to3/__init__.py b/contrib/python/black/blib2to3/__init__.py new file mode 100644 index 00000000000..1bb8bf6d7fd --- /dev/null +++ b/contrib/python/black/blib2to3/__init__.py @@ -0,0 +1 @@ +# empty diff --git a/contrib/python/black/blib2to3/pgen2/__init__.py b/contrib/python/black/blib2to3/pgen2/__init__.py new file mode 100644 index 00000000000..af390484528 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""The pgen2 package.""" diff --git a/contrib/python/black/blib2to3/pgen2/conv.py b/contrib/python/black/blib2to3/pgen2/conv.py new file mode 100644 index 00000000000..5534ef3b969 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/conv.py @@ -0,0 +1,256 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# mypy: ignore-errors + +"""Convert graminit.[ch] spit out by pgen to Python code. + +Pgen is the Python parser generator. It is useful to quickly create a +parser from a grammar file in Python's grammar notation. But I don't +want my parsers to be written in C (yet), so I'm translating the +parsing tables to Python data structures and writing a Python parse +engine. + +Note that the token numbers are constants determined by the standard +Python tokenizer. The standard token module defines these numbers and +their names (the names are not used much). The token numbers are +hardcoded into the Python tokenizer and into pgen. A Python +implementation of the Python tokenizer is also available, in the +standard tokenize module. + +On the other hand, symbol numbers (representing the grammar's +non-terminals) are assigned by pgen based on the actual grammar +input. + +Note: this module is pretty much obsolete; the pgen module generates +equivalent grammar tables directly from the Grammar.txt input file +without having to invoke the Python pgen C program. + +""" + +# Python imports +import re + +# Local imports +from ..pgen2 import grammar, token + + +class Converter(grammar.Grammar): + """Grammar subclass that reads classic pgen output files. + + The run() method reads the tables as produced by the pgen parser + generator, typically contained in two C files, graminit.h and + graminit.c. The other methods are for internal use only. + + See the base class for more documentation. + + """ + + def run(self, graminit_h, graminit_c): + """Load the grammar tables from the text files written by pgen.""" + self.parse_graminit_h(graminit_h) + self.parse_graminit_c(graminit_c) + self.finish_off() + + def parse_graminit_h(self, filename): + """Parse the .h file written by pgen. (Internal) + + This file is a sequence of #define statements defining the + nonterminals of the grammar as numbers. We build two tables + mapping the numbers to names and back. + + """ + try: + f = open(filename) + except OSError as err: + print(f"Can't open {filename}: {err}") + return False + self.symbol2number = {} + self.number2symbol = {} + lineno = 0 + for line in f: + lineno += 1 + mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) + if not mo and line.strip(): + print(f"{filename}({lineno}): can't parse {line.strip()}") + else: + symbol, number = mo.groups() + number = int(number) + assert symbol not in self.symbol2number + assert number not in self.number2symbol + self.symbol2number[symbol] = number + self.number2symbol[number] = symbol + return True + + def parse_graminit_c(self, filename): + """Parse the .c file written by pgen. (Internal) + + The file looks as follows. The first two lines are always this: + + #include "pgenheaders.h" + #include "grammar.h" + + After that come four blocks: + + 1) one or more state definitions + 2) a table defining dfas + 3) a table defining labels + 4) a struct defining the grammar + + A state definition has the following form: + - one or more arc arrays, each of the form: + static arc arcs_<n>_<m>[<k>] = { + {<i>, <j>}, + ... + }; + - followed by a state array, of the form: + static state states_<s>[<t>] = { + {<k>, arcs_<n>_<m>}, + ... + }; + + """ + try: + f = open(filename) + except OSError as err: + print(f"Can't open {filename}: {err}") + return False + # The code below essentially uses f's iterator-ness! + lineno = 0 + + # Expect the two #include lines + lineno, line = lineno + 1, next(f) + assert line == '#include "pgenheaders.h"\n', (lineno, line) + lineno, line = lineno + 1, next(f) + assert line == '#include "grammar.h"\n', (lineno, line) + + # Parse the state definitions + lineno, line = lineno + 1, next(f) + allarcs = {} + states = [] + while line.startswith("static arc "): + while line.startswith("static arc "): + mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", line) + assert mo, (lineno, line) + n, m, k = list(map(int, mo.groups())) + arcs = [] + for _ in range(k): + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+{(\d+), (\d+)},$", line) + assert mo, (lineno, line) + i, j = list(map(int, mo.groups())) + arcs.append((i, j)) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + allarcs[(n, m)] = arcs + lineno, line = lineno + 1, next(f) + mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line) + assert mo, (lineno, line) + s, t = list(map(int, mo.groups())) + assert s == len(states), (lineno, line) + state = [] + for _ in range(t): + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line) + assert mo, (lineno, line) + k, n, m = list(map(int, mo.groups())) + arcs = allarcs[n, m] + assert k == len(arcs), (lineno, line) + state.append(arcs) + states.append(state) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + lineno, line = lineno + 1, next(f) + self.states = states + + # Parse the dfas + dfas = {} + mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line) + assert mo, (lineno, line) + ndfas = int(mo.group(1)) + for i in range(ndfas): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', line) + assert mo, (lineno, line) + symbol = mo.group(2) + number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) + assert self.symbol2number[symbol] == number, (lineno, line) + assert self.number2symbol[number] == symbol, (lineno, line) + assert x == 0, (lineno, line) + state = states[z] + assert y == len(state), (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) + assert mo, (lineno, line) + first = {} + rawbitset = eval(mo.group(1)) + for i, c in enumerate(rawbitset): + byte = ord(c) + for j in range(8): + if byte & (1 << j): + first[i * 8 + j] = 1 + dfas[number] = (state, first) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + self.dfas = dfas + + # Parse the labels + labels = [] + lineno, line = lineno + 1, next(f) + mo = re.match(r"static label labels\[(\d+)\] = {$", line) + assert mo, (lineno, line) + nlabels = int(mo.group(1)) + for i in range(nlabels): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line) + assert mo, (lineno, line) + x, y = mo.groups() + x = int(x) + if y == "0": + y = None + else: + y = eval(y) + labels.append((x, y)) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + self.labels = labels + + # Parse the grammar struct + lineno, line = lineno + 1, next(f) + assert line == "grammar _PyParser_Grammar = {\n", (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+(\d+),$", line) + assert mo, (lineno, line) + ndfas = int(mo.group(1)) + assert ndfas == len(self.dfas) + lineno, line = lineno + 1, next(f) + assert line == "\tdfas,\n", (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+{(\d+), labels},$", line) + assert mo, (lineno, line) + nlabels = int(mo.group(1)) + assert nlabels == len(self.labels), (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+(\d+)$", line) + assert mo, (lineno, line) + start = int(mo.group(1)) + assert start in self.number2symbol, (lineno, line) + self.start = start + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + try: + lineno, line = lineno + 1, next(f) + except StopIteration: + pass + else: + assert 0, (lineno, line) + + def finish_off(self): + """Create additional useful structures. (Internal).""" + self.keywords = {} # map from keyword strings to arc labels + self.tokens = {} # map from numeric token values to arc labels + for ilabel, (type, value) in enumerate(self.labels): + if type == token.NAME and value is not None: + self.keywords[value] = ilabel + elif value is None: + self.tokens[type] = ilabel diff --git a/contrib/python/black/blib2to3/pgen2/driver.py b/contrib/python/black/blib2to3/pgen2/driver.py new file mode 100644 index 00000000000..486ac6150e6 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/driver.py @@ -0,0 +1,325 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# Modifications: +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Parser driver. + +This provides a high-level interface to parse a file into a syntax tree. + +""" + +__author__ = "Guido van Rossum <[email protected]>" + +__all__ = ["Driver", "load_grammar"] + +# Python imports +import io +import logging +import os +import pkgutil +import sys +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from logging import Logger +from typing import IO, Any, Optional, Union, cast + +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.tokenize import GoodTokenInfo +from blib2to3.pytree import NL + +# Pgen imports +from . import grammar, parse, pgen, token, tokenize + +Path = Union[str, "os.PathLike[str]"] + +try: + import __res + IS_ARCADIA = True +except ImportError: + IS_ARCADIA = False + + +@dataclass +class ReleaseRange: + start: int + end: Optional[int] = None + tokens: list[Any] = field(default_factory=list) + + def lock(self) -> None: + total_eaten = len(self.tokens) + self.end = self.start + total_eaten + + +class TokenProxy: + def __init__(self, generator: Any) -> None: + self._tokens = generator + self._counter = 0 + self._release_ranges: list[ReleaseRange] = [] + + @contextmanager + def release(self) -> Iterator["TokenProxy"]: + release_range = ReleaseRange(self._counter) + self._release_ranges.append(release_range) + try: + yield self + finally: + # Lock the last release range to the final position that + # has been eaten. + release_range.lock() + + def eat(self, point: int) -> Any: + eaten_tokens = self._release_ranges[-1].tokens + if point < len(eaten_tokens): + return eaten_tokens[point] + else: + while point >= len(eaten_tokens): + token = next(self._tokens) + eaten_tokens.append(token) + return token + + def __iter__(self) -> "TokenProxy": + return self + + def __next__(self) -> Any: + # If the current position is already compromised (looked up) + # return the eaten token, if not just go further on the given + # token producer. + for release_range in self._release_ranges: + assert release_range.end is not None + + start, end = release_range.start, release_range.end + if start <= self._counter < end: + token = release_range.tokens[self._counter - start] + break + else: + token = next(self._tokens) + self._counter += 1 + return token + + def can_advance(self, to: int) -> bool: + # Try to eat, fail if it can't. The eat operation is cached + # so there won't be any additional cost of eating here + try: + self.eat(to) + except StopIteration: + return False + else: + return True + + +class Driver: + def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None: + self.grammar = grammar + if logger is None: + logger = logging.getLogger(__name__) + self.logger = logger + + def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> NL: + """Parse a series of tokens and return the syntax tree.""" + # XXX Move the prefix computation into a wrapper around tokenize. + proxy = TokenProxy(tokens) + + p = parse.Parser(self.grammar) + p.setup(proxy=proxy) + + lineno = 1 + column = 0 + indent_columns: list[int] = [] + type = value = start = end = line_text = None + prefix = "" + + for quintuple in proxy: + type, value, start, end, line_text = quintuple + if start != (lineno, column): + assert (lineno, column) <= start, ((lineno, column), start) + s_lineno, s_column = start + if lineno < s_lineno: + prefix += "\n" * (s_lineno - lineno) + lineno = s_lineno + column = 0 + if column < s_column: + prefix += line_text[column:s_column] + column = s_column + if type in (tokenize.COMMENT, tokenize.NL): + prefix += value + lineno, column = end + if value.endswith("\n"): + lineno += 1 + column = 0 + continue + if type == token.OP: + type = grammar.opmap[value] + if debug: + assert type is not None + self.logger.debug( + "%s %r (prefix=%r)", token.tok_name[type], value, prefix + ) + if type == token.INDENT: + indent_columns.append(len(value)) + _prefix = prefix + value + prefix = "" + value = "" + elif type == token.DEDENT: + _indent_col = indent_columns.pop() + prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col) + if p.addtoken(cast(int, type), value, (prefix, start)): + if debug: + self.logger.debug("Stop.") + break + prefix = "" + if type in {token.INDENT, token.DEDENT}: + prefix = _prefix + lineno, column = end + # FSTRING_MIDDLE is the only token that can end with a newline, and + # `end` will point to the next line. For that case, don't increment lineno. + if value.endswith("\n") and type != token.FSTRING_MIDDLE: + lineno += 1 + column = 0 + else: + # We never broke out -- EOF is too soon (how can this happen???) + assert start is not None + raise parse.ParseError("incomplete input", type, value, (prefix, start)) + assert p.rootnode is not None + return p.rootnode + + def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> NL: + """Parse a stream and return the syntax tree.""" + tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar) + return self.parse_tokens(tokens, debug) + + def parse_stream(self, stream: IO[str], debug: bool = False) -> NL: + """Parse a stream and return the syntax tree.""" + return self.parse_stream_raw(stream, debug) + + def parse_file( + self, filename: Path, encoding: Optional[str] = None, debug: bool = False + ) -> NL: + """Parse a file and return the syntax tree.""" + with open(filename, encoding=encoding) as stream: + return self.parse_stream(stream, debug) + + def parse_string(self, text: str, debug: bool = False) -> NL: + """Parse a string and return the syntax tree.""" + tokens = tokenize.generate_tokens( + io.StringIO(text).readline, grammar=self.grammar + ) + return self.parse_tokens(tokens, debug) + + def _partially_consume_prefix(self, prefix: str, column: int) -> tuple[str, str]: + lines: list[str] = [] + current_line = "" + current_column = 0 + wait_for_nl = False + for char in prefix: + current_line += char + if wait_for_nl: + if char == "\n": + if current_line.strip() and current_column < column: + res = "".join(lines) + return res, prefix[len(res) :] + + lines.append(current_line) + current_line = "" + current_column = 0 + wait_for_nl = False + elif char in " \t": + current_column += 1 + elif char == "\n": + # unexpected empty line + current_column = 0 + elif char == "\f": + current_column = 0 + else: + # indent is finished + wait_for_nl = True + return "".join(lines), current_line + + +def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> str: + head, tail = os.path.splitext(gt) + if tail == ".txt": + tail = "" + name = head + tail + ".".join(map(str, sys.version_info)) + ".pickle" + if cache_dir: + return os.path.join(cache_dir, os.path.basename(name)) + else: + return name + + +def load_grammar( + gt: str = "Grammar.txt", + gp: Optional[str] = None, + save: bool = True, + force: bool = False, + logger: Optional[Logger] = None, +) -> Grammar: + """Load the grammar (maybe from a pickle).""" + if logger is None: + logger = logging.getLogger(__name__) + gp = _generate_pickle_name(gt) if gp is None else gp + if force or not _newer(gp, gt): + g: grammar.Grammar = pgen.generate_grammar(gt) + if save: + try: + g.dump(gp) + except OSError: + # Ignore error, caching is not vital. + pass + else: + g = grammar.Grammar() + g.load(gp) + return g + + +def _newer(a: str, b: str) -> bool: + """Inquire whether file a was written since file b.""" + if not os.path.exists(a): + return False + if not os.path.exists(b): + return True + return os.path.getmtime(a) >= os.path.getmtime(b) + + +def load_packaged_grammar( + package: str, grammar_source: str, cache_dir: Optional[Path] = None +) -> grammar.Grammar: + """Normally, loads a pickled grammar by doing + pkgutil.get_data(package, pickled_grammar) + where *pickled_grammar* is computed from *grammar_source* by adding the + Python version and using a ``.pickle`` extension. + + However, if *grammar_source* is an extant file, load_grammar(grammar_source) + is called instead. This facilitates using a packaged grammar file when needed + but preserves load_grammar's automatic regeneration behavior when possible. + + """ + if os.path.isfile(grammar_source) or IS_ARCADIA: + gp = _generate_pickle_name(grammar_source, cache_dir) if cache_dir else None + return load_grammar(grammar_source, gp=gp) + pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir) + data = pkgutil.get_data(package, pickled_name) + assert data is not None + g = grammar.Grammar() + g.loads(data) + return g + + +def main(*args: str) -> bool: + """Main program, when run as a script: produce grammar pickle files. + + Calls load_grammar for each argument, a path to a grammar text file. + """ + if not args: + args = tuple(sys.argv[1:]) + logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s") + for gt in args: + load_grammar(gt, save=True, force=True) + return True + + +if __name__ == "__main__": + sys.exit(int(not main())) diff --git a/contrib/python/black/blib2to3/pgen2/grammar.py b/contrib/python/black/blib2to3/pgen2/grammar.py new file mode 100644 index 00000000000..9cf24037754 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/grammar.py @@ -0,0 +1,228 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""This module defines the data structures used to represent a grammar. + +These are a bit arcane because they are derived from the data +structures used by Python's 'pgen' parser generator. + +There's also a table here mapping operators to their names in the +token module; the Python tokenize module reports all operators as the +fallback token code OP, but the parser needs the actual token code. + +""" + +# Python imports +import os +import pickle +import tempfile +from typing import Any, Optional, TypeVar, Union + +# Local imports +from . import token + +_P = TypeVar("_P", bound="Grammar") +Label = tuple[int, Optional[str]] +DFA = list[list[tuple[int, int]]] +DFAS = tuple[DFA, dict[int, int]] +Path = Union[str, "os.PathLike[str]"] + + +class Grammar: + """Pgen parsing tables conversion class. + + Once initialized, this class supplies the grammar tables for the + parsing engine implemented by parse.py. The parsing engine + accesses the instance variables directly. The class here does not + provide initialization of the tables; several subclasses exist to + do this (see the conv and pgen modules). + + The load() method reads the tables from a pickle file, which is + much faster than the other ways offered by subclasses. The pickle + file is written by calling dump() (after loading the grammar + tables using a subclass). The report() method prints a readable + representation of the tables to stdout, for debugging. + + The instance variables are as follows: + + symbol2number -- a dict mapping symbol names to numbers. Symbol + numbers are always 256 or higher, to distinguish + them from token numbers, which are between 0 and + 255 (inclusive). + + number2symbol -- a dict mapping numbers to symbol names; + these two are each other's inverse. + + states -- a list of DFAs, where each DFA is a list of + states, each state is a list of arcs, and each + arc is a (i, j) pair where i is a label and j is + a state number. The DFA number is the index into + this list. (This name is slightly confusing.) + Final states are represented by a special arc of + the form (0, j) where j is its own state number. + + dfas -- a dict mapping symbol numbers to (DFA, first) + pairs, where DFA is an item from the states list + above, and first is a set of tokens that can + begin this grammar rule (represented by a dict + whose values are always 1). + + labels -- a list of (x, y) pairs where x is either a token + number or a symbol number, and y is either None + or a string; the strings are keywords. The label + number is the index in this list; label numbers + are used to mark state transitions (arcs) in the + DFAs. + + start -- the number of the grammar's start symbol. + + keywords -- a dict mapping keyword strings to arc labels. + + tokens -- a dict mapping token numbers to arc labels. + + """ + + def __init__(self) -> None: + self.symbol2number: dict[str, int] = {} + self.number2symbol: dict[int, str] = {} + self.states: list[DFA] = [] + self.dfas: dict[int, DFAS] = {} + self.labels: list[Label] = [(0, "EMPTY")] + self.keywords: dict[str, int] = {} + self.soft_keywords: dict[str, int] = {} + self.tokens: dict[int, int] = {} + self.symbol2label: dict[str, int] = {} + self.version: tuple[int, int] = (0, 0) + self.start = 256 + # Python 3.7+ parses async as a keyword, not an identifier + self.async_keywords = False + + def dump(self, filename: Path) -> None: + """Dump the grammar tables to a pickle file.""" + + # mypyc generates objects that don't have a __dict__, but they + # do have __getstate__ methods that will return an equivalent + # dictionary + if hasattr(self, "__dict__"): + d = self.__dict__ + else: + d = self.__getstate__() # type: ignore + + with tempfile.NamedTemporaryFile( + dir=os.path.dirname(filename), delete=False + ) as f: + pickle.dump(d, f, pickle.HIGHEST_PROTOCOL) + os.replace(f.name, filename) + + def _update(self, attrs: dict[str, Any]) -> None: + for k, v in attrs.items(): + setattr(self, k, v) + + def load(self, filename: Path) -> None: + """Load the grammar tables from a pickle file.""" + with open(filename, "rb") as f: + d = pickle.load(f) + self._update(d) + + def loads(self, pkl: bytes) -> None: + """Load the grammar tables from a pickle bytes object.""" + self._update(pickle.loads(pkl)) + + def copy(self: _P) -> _P: + """ + Copy the grammar. + """ + new = self.__class__() + for dict_attr in ( + "symbol2number", + "number2symbol", + "dfas", + "keywords", + "soft_keywords", + "tokens", + "symbol2label", + ): + setattr(new, dict_attr, getattr(self, dict_attr).copy()) + new.labels = self.labels[:] + new.states = self.states[:] + new.start = self.start + new.version = self.version + new.async_keywords = self.async_keywords + return new + + def report(self) -> None: + """Dump the grammar tables to standard output, for debugging.""" + from pprint import pprint + + print("s2n") + pprint(self.symbol2number) + print("n2s") + pprint(self.number2symbol) + print("states") + pprint(self.states) + print("dfas") + pprint(self.dfas) + print("labels") + pprint(self.labels) + print("start", self.start) + + +# Map from operator to number (since tokenize doesn't do this) + +opmap_raw = """ +( LPAR +) RPAR +[ LSQB +] RSQB +: COLON +, COMMA +; SEMI ++ PLUS +- MINUS +* STAR +/ SLASH +| VBAR +& AMPER +< LESS +> GREATER += EQUAL +. DOT +% PERCENT +` BACKQUOTE +{ LBRACE +} RBRACE +@ AT +@= ATEQUAL +== EQEQUAL +!= NOTEQUAL +<> NOTEQUAL +<= LESSEQUAL +>= GREATEREQUAL +~ TILDE +^ CIRCUMFLEX +<< LEFTSHIFT +>> RIGHTSHIFT +** DOUBLESTAR ++= PLUSEQUAL +-= MINEQUAL +*= STAREQUAL +/= SLASHEQUAL +%= PERCENTEQUAL +&= AMPEREQUAL +|= VBAREQUAL +^= CIRCUMFLEXEQUAL +<<= LEFTSHIFTEQUAL +>>= RIGHTSHIFTEQUAL +**= DOUBLESTAREQUAL +// DOUBLESLASH +//= DOUBLESLASHEQUAL +-> RARROW +:= COLONEQUAL +! BANG +""" + +opmap = {} +for line in opmap_raw.splitlines(): + if line: + op, name = line.split() + opmap[op] = getattr(token, name) diff --git a/contrib/python/black/blib2to3/pgen2/literals.py b/contrib/python/black/blib2to3/pgen2/literals.py new file mode 100644 index 00000000000..a738c10f460 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/literals.py @@ -0,0 +1,65 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Safely evaluate Python string literals without using eval().""" + +import re + +simple_escapes: dict[str, str] = { + "a": "\a", + "b": "\b", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", + "'": "'", + '"': '"', + "\\": "\\", +} + + +def escape(m: re.Match[str]) -> str: + all, tail = m.group(0, 1) + assert all.startswith("\\") + esc = simple_escapes.get(tail) + if esc is not None: + return esc + if tail.startswith("x"): + hexes = tail[1:] + if len(hexes) < 2: + raise ValueError("invalid hex string escape ('\\%s')" % tail) + try: + i = int(hexes, 16) + except ValueError: + raise ValueError("invalid hex string escape ('\\%s')" % tail) from None + else: + try: + i = int(tail, 8) + except ValueError: + raise ValueError("invalid octal string escape ('\\%s')" % tail) from None + return chr(i) + + +def evalString(s: str) -> str: + assert s.startswith("'") or s.startswith('"'), repr(s[:1]) + q = s[0] + if s[:3] == q * 3: + q = q * 3 + assert s.endswith(q), repr(s[-len(q) :]) + assert len(s) >= 2 * len(q) + s = s[len(q) : -len(q)] + return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) + + +def test() -> None: + for i in range(256): + c = chr(i) + s = repr(c) + e = evalString(s) + if e != c: + print(i, c, s, e) + + +if __name__ == "__main__": + test() diff --git a/contrib/python/black/blib2to3/pgen2/parse.py b/contrib/python/black/blib2to3/pgen2/parse.py new file mode 100644 index 00000000000..10202ab6002 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/parse.py @@ -0,0 +1,400 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Parser engine for the grammar tables generated by pgen. + +The grammar table must be loaded first. + +See Parser/parser.c in the Python distribution for additional info on +how this parsing engine works. + +""" +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Optional, Union, cast + +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pytree import NL, Context, Leaf, Node, RawNode, convert + +# Local imports +from . import grammar, token, tokenize + +if TYPE_CHECKING: + from blib2to3.pgen2.driver import TokenProxy + + +Results = dict[str, NL] +Convert = Callable[[Grammar, RawNode], Union[Node, Leaf]] +DFA = list[list[tuple[int, int]]] +DFAS = tuple[DFA, dict[int, int]] + + +def lam_sub(grammar: Grammar, node: RawNode) -> NL: + assert node[3] is not None + return Node(type=node[0], children=node[3], context=node[2]) + + +# A placeholder node, used when parser is backtracking. +DUMMY_NODE = (-1, None, None, None) + + +def stack_copy( + stack: list[tuple[DFAS, int, RawNode]], +) -> list[tuple[DFAS, int, RawNode]]: + """Nodeless stack copy.""" + return [(dfa, label, DUMMY_NODE) for dfa, label, _ in stack] + + +class Recorder: + def __init__(self, parser: "Parser", ilabels: list[int], context: Context) -> None: + self.parser = parser + self._ilabels = ilabels + self.context = context # not really matter + + self._dead_ilabels: set[int] = set() + self._start_point = self.parser.stack + self._points = {ilabel: stack_copy(self._start_point) for ilabel in ilabels} + + @property + def ilabels(self) -> set[int]: + return self._dead_ilabels.symmetric_difference(self._ilabels) + + @contextmanager + def switch_to(self, ilabel: int) -> Iterator[None]: + with self.backtrack(): + self.parser.stack = self._points[ilabel] + try: + yield + except ParseError: + self._dead_ilabels.add(ilabel) + finally: + self.parser.stack = self._start_point + + @contextmanager + def backtrack(self) -> Iterator[None]: + """ + Use the node-level invariant ones for basic parsing operations (push/pop/shift). + These still will operate on the stack; but they won't create any new nodes, or + modify the contents of any other existing nodes. + + This saves us a ton of time when we are backtracking, since we + want to restore to the initial state as quick as possible, which + can only be done by having as little mutatations as possible. + """ + is_backtracking = self.parser.is_backtracking + try: + self.parser.is_backtracking = True + yield + finally: + self.parser.is_backtracking = is_backtracking + + def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None: + func: Callable[..., Any] + if raw: + func = self.parser._addtoken + else: + func = self.parser.addtoken + + for ilabel in self.ilabels: + with self.switch_to(ilabel): + args = [tok_type, tok_val, self.context] + if raw: + args.insert(0, ilabel) + func(*args) + + def determine_route( + self, value: Optional[str] = None, force: bool = False + ) -> Optional[int]: + alive_ilabels = self.ilabels + if len(alive_ilabels) == 0: + *_, most_successful_ilabel = self._dead_ilabels + raise ParseError("bad input", most_successful_ilabel, value, self.context) + + ilabel, *rest = alive_ilabels + if force or not rest: + return ilabel + else: + return None + + +class ParseError(Exception): + """Exception to signal the parser is stuck.""" + + def __init__( + self, msg: str, type: Optional[int], value: Optional[str], context: Context + ) -> None: + Exception.__init__( + self, f"{msg}: type={type!r}, value={value!r}, context={context!r}" + ) + self.msg = msg + self.type = type + self.value = value + self.context = context + + +class Parser: + """Parser engine. + + The proper usage sequence is: + + p = Parser(grammar, [converter]) # create instance + p.setup([start]) # prepare for parsing + <for each input token>: + if p.addtoken(...): # parse a token; may raise ParseError + break + root = p.rootnode # root of abstract syntax tree + + A Parser instance may be reused by calling setup() repeatedly. + + A Parser instance contains state pertaining to the current token + sequence, and should not be used concurrently by different threads + to parse separate token sequences. + + See driver.py for how to get input tokens by tokenizing a file or + string. + + Parsing is complete when addtoken() returns True; the root of the + abstract syntax tree can then be retrieved from the rootnode + instance variable. When a syntax error occurs, addtoken() raises + the ParseError exception. There is no error recovery; the parser + cannot be used after a syntax error was reported (but it can be + reinitialized by calling setup()). + + """ + + def __init__(self, grammar: Grammar, convert: Optional[Convert] = None) -> None: + """Constructor. + + The grammar argument is a grammar.Grammar instance; see the + grammar module for more information. + + The parser is not ready yet for parsing; you must call the + setup() method to get it started. + + The optional convert argument is a function mapping concrete + syntax tree nodes to abstract syntax tree nodes. If not + given, no conversion is done and the syntax tree produced is + the concrete syntax tree. If given, it must be a function of + two arguments, the first being the grammar (a grammar.Grammar + instance), and the second being the concrete syntax tree node + to be converted. The syntax tree is converted from the bottom + up. + + **post-note: the convert argument is ignored since for Black's + usage, convert will always be blib2to3.pytree.convert. Allowing + this to be dynamic hurts mypyc's ability to use early binding. + These docs are left for historical and informational value. + + A concrete syntax tree node is a (type, value, context, nodes) + tuple, where type is the node type (a token or symbol number), + value is None for symbols and a string for tokens, context is + None or an opaque value used for error reporting (typically a + (lineno, offset) pair), and nodes is a list of children for + symbols, and None for tokens. + + An abstract syntax tree node may be anything; this is entirely + up to the converter function. + + """ + self.grammar = grammar + # See note in docstring above. TL;DR this is ignored. + self.convert = convert or lam_sub + self.is_backtracking = False + self.last_token: Optional[int] = None + + def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: + """Prepare for parsing. + + This *must* be called before starting to parse. + + The optional argument is an alternative start symbol; it + defaults to the grammar's start symbol. + + You can use a Parser instance to parse any number of programs; + each time you call setup() the parser is reset to an initial + state determined by the (implicit or explicit) start symbol. + + """ + if start is None: + start = self.grammar.start + # Each stack entry is a tuple: (dfa, state, node). + # A node is a tuple: (type, value, context, children), + # where children is a list of nodes or None, and context may be None. + newnode: RawNode = (start, None, None, []) + stackentry = (self.grammar.dfas[start], 0, newnode) + self.stack: list[tuple[DFAS, int, RawNode]] = [stackentry] + self.rootnode: Optional[NL] = None + self.used_names: set[str] = set() + self.proxy = proxy + self.last_token = None + + def addtoken(self, type: int, value: str, context: Context) -> bool: + """Add a token; return True iff this is the end of the program.""" + # Map from token to label + ilabels = self.classify(type, value, context) + assert len(ilabels) >= 1 + + # If we have only one state to advance, we'll directly + # take it as is. + if len(ilabels) == 1: + [ilabel] = ilabels + return self._addtoken(ilabel, type, value, context) + + # If there are multiple states which we can advance (only + # happen under soft-keywords), then we will try all of them + # in parallel and as soon as one state can reach further than + # the rest, we'll choose that one. This is a pretty hacky + # and hopefully temporary algorithm. + # + # For a more detailed explanation, check out this post: + # https://tree.science/what-the-backtracking.html + + with self.proxy.release() as proxy: + counter, force = 0, False + recorder = Recorder(self, ilabels, context) + recorder.add_token(type, value, raw=True) + + next_token_value = value + while recorder.determine_route(next_token_value) is None: + if not proxy.can_advance(counter): + force = True + break + + next_token_type, next_token_value, *_ = proxy.eat(counter) + if next_token_type in (tokenize.COMMENT, tokenize.NL): + counter += 1 + continue + + if next_token_type == tokenize.OP: + next_token_type = grammar.opmap[next_token_value] + + recorder.add_token(next_token_type, next_token_value) + counter += 1 + + ilabel = cast(int, recorder.determine_route(next_token_value, force=force)) + assert ilabel is not None + + return self._addtoken(ilabel, type, value, context) + + def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> bool: + # Loop until the token is shifted; may raise exceptions + while True: + dfa, state, node = self.stack[-1] + states, first = dfa + arcs = states[state] + # Look for a state with this label + for i, newstate in arcs: + t = self.grammar.labels[i][0] + if t >= 256: + # See if it's a symbol and if we're in its first set + itsdfa = self.grammar.dfas[t] + itsstates, itsfirst = itsdfa + if ilabel in itsfirst: + # Push a symbol + self.push(t, itsdfa, newstate, context) + break # To continue the outer while loop + + elif ilabel == i: + # Look it up in the list of labels + # Shift a token; we're done with it + self.shift(type, value, newstate, context) + # Pop while we are in an accept-only state + state = newstate + while states[state] == [(0, state)]: + self.pop() + if not self.stack: + # Done parsing! + return True + dfa, state, node = self.stack[-1] + states, first = dfa + # Done with this token + self.last_token = type + return False + + else: + if (0, state) in arcs: + # An accepting state, pop it and try something else + self.pop() + if not self.stack: + # Done parsing, but another token is input + raise ParseError("too much input", type, value, context) + else: + # No success finding a transition + raise ParseError("bad input", type, value, context) + + def classify(self, type: int, value: str, context: Context) -> list[int]: + """Turn a token into a label. (Internal) + + Depending on whether the value is a soft-keyword or not, + this function may return multiple labels to choose from.""" + if type == token.NAME: + # Keep a listing of all used names + self.used_names.add(value) + # Check for reserved words + if value in self.grammar.keywords: + return [self.grammar.keywords[value]] + elif value in self.grammar.soft_keywords: + assert type in self.grammar.tokens + # Current soft keywords (match, case, type) can only appear at the + # beginning of a statement. So as a shortcut, don't try to treat them + # like keywords in any other context. + # ('_' is also a soft keyword in the real grammar, but for our grammar + # it's just an expression, so we don't need to treat it specially.) + if self.last_token not in ( + None, + token.INDENT, + token.DEDENT, + token.NEWLINE, + token.SEMI, + token.COLON, + ): + return [self.grammar.tokens[type]] + return [ + self.grammar.tokens[type], + self.grammar.soft_keywords[value], + ] + + ilabel = self.grammar.tokens.get(type) + if ilabel is None: + raise ParseError("bad token", type, value, context) + return [ilabel] + + def shift(self, type: int, value: str, newstate: int, context: Context) -> None: + """Shift a token. (Internal)""" + if self.is_backtracking: + dfa, state, _ = self.stack[-1] + self.stack[-1] = (dfa, newstate, DUMMY_NODE) + else: + dfa, state, node = self.stack[-1] + rawnode: RawNode = (type, value, context, None) + newnode = convert(self.grammar, rawnode) + assert node[-1] is not None + node[-1].append(newnode) + self.stack[-1] = (dfa, newstate, node) + + def push(self, type: int, newdfa: DFAS, newstate: int, context: Context) -> None: + """Push a nonterminal. (Internal)""" + if self.is_backtracking: + dfa, state, _ = self.stack[-1] + self.stack[-1] = (dfa, newstate, DUMMY_NODE) + self.stack.append((newdfa, 0, DUMMY_NODE)) + else: + dfa, state, node = self.stack[-1] + newnode: RawNode = (type, None, context, []) + self.stack[-1] = (dfa, newstate, node) + self.stack.append((newdfa, 0, newnode)) + + def pop(self) -> None: + """Pop a nonterminal. (Internal)""" + if self.is_backtracking: + self.stack.pop() + else: + popdfa, popstate, popnode = self.stack.pop() + newnode = convert(self.grammar, popnode) + if self.stack: + dfa, state, node = self.stack[-1] + assert node[-1] is not None + node[-1].append(newnode) + else: + self.rootnode = newnode + self.rootnode.used_names = self.used_names diff --git a/contrib/python/black/blib2to3/pgen2/pgen.py b/contrib/python/black/blib2to3/pgen2/pgen.py new file mode 100644 index 00000000000..692e9db28f9 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/pgen.py @@ -0,0 +1,427 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +import os +from collections.abc import Iterator, Sequence +from typing import IO, Any, NoReturn, Optional, Union + +from blib2to3.pgen2 import grammar, token, tokenize +from blib2to3.pgen2.tokenize import GoodTokenInfo + +Path = Union[str, "os.PathLike[str]"] + +import pkgutil +import io + + +class PgenGrammar(grammar.Grammar): + pass + + +class ParserGenerator: + filename: Path + stream: IO[str] + generator: Iterator[GoodTokenInfo] + first: dict[str, Optional[dict[str, int]]] + + def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None: + close_stream = None + if stream is None: + data = pkgutil.get_data("blib2to3", os.path.basename(filename)) + if data: + stream = io.StringIO(data.decode("utf-8")) + else: + stream = open(filename, encoding="utf-8") + close_stream = stream.close + self.filename = filename + self.stream = stream + self.generator = tokenize.generate_tokens(stream.readline) + self.gettoken() # Initialize lookahead + self.dfas, self.startsymbol = self.parse() + if close_stream is not None: + close_stream() + self.first = {} # map from symbol name to set of tokens + self.addfirstsets() + + def make_grammar(self) -> PgenGrammar: + c = PgenGrammar() + names = list(self.dfas.keys()) + names.sort() + names.remove(self.startsymbol) + names.insert(0, self.startsymbol) + for name in names: + i = 256 + len(c.symbol2number) + c.symbol2number[name] = i + c.number2symbol[i] = name + for name in names: + dfa = self.dfas[name] + states = [] + for state in dfa: + arcs = [] + for label, next in sorted(state.arcs.items()): + arcs.append((self.make_label(c, label), dfa.index(next))) + if state.isfinal: + arcs.append((0, dfa.index(state))) + states.append(arcs) + c.states.append(states) + c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) + c.start = c.symbol2number[self.startsymbol] + return c + + def make_first(self, c: PgenGrammar, name: str) -> dict[int, int]: + rawfirst = self.first[name] + assert rawfirst is not None + first = {} + for label in sorted(rawfirst): + ilabel = self.make_label(c, label) + ##assert ilabel not in first # XXX failed on <> ... != + first[ilabel] = 1 + return first + + def make_label(self, c: PgenGrammar, label: str) -> int: + # XXX Maybe this should be a method on a subclass of converter? + ilabel = len(c.labels) + if label[0].isalpha(): + # Either a symbol name or a named token + if label in c.symbol2number: + # A symbol name (a non-terminal) + if label in c.symbol2label: + return c.symbol2label[label] + else: + c.labels.append((c.symbol2number[label], None)) + c.symbol2label[label] = ilabel + return ilabel + else: + # A named token (NAME, NUMBER, STRING) + itoken = getattr(token, label, None) + assert isinstance(itoken, int), label + assert itoken in token.tok_name, label + if itoken in c.tokens: + return c.tokens[itoken] + else: + c.labels.append((itoken, None)) + c.tokens[itoken] = ilabel + return ilabel + else: + # Either a keyword or an operator + assert label[0] in ('"', "'"), label + value = eval(label) + if value[0].isalpha(): + if label[0] == '"': + keywords = c.soft_keywords + else: + keywords = c.keywords + + # A keyword + if value in keywords: + return keywords[value] + else: + c.labels.append((token.NAME, value)) + keywords[value] = ilabel + return ilabel + else: + # An operator (any non-numeric token) + itoken = grammar.opmap[value] # Fails if unknown token + if itoken in c.tokens: + return c.tokens[itoken] + else: + c.labels.append((itoken, None)) + c.tokens[itoken] = ilabel + return ilabel + + def addfirstsets(self) -> None: + names = list(self.dfas.keys()) + names.sort() + for name in names: + if name not in self.first: + self.calcfirst(name) + # print name, self.first[name].keys() + + def calcfirst(self, name: str) -> None: + dfa = self.dfas[name] + self.first[name] = None # dummy to detect left recursion + state = dfa[0] + totalset: dict[str, int] = {} + overlapcheck = {} + for label in state.arcs: + if label in self.dfas: + if label in self.first: + fset = self.first[label] + if fset is None: + raise ValueError("recursion for rule %r" % name) + else: + self.calcfirst(label) + fset = self.first[label] + assert fset is not None + totalset.update(fset) + overlapcheck[label] = fset + else: + totalset[label] = 1 + overlapcheck[label] = {label: 1} + inverse: dict[str, str] = {} + for label, itsfirst in overlapcheck.items(): + for symbol in itsfirst: + if symbol in inverse: + raise ValueError( + "rule %s is ambiguous; %s is in the first sets of %s as well" + " as %s" % (name, symbol, label, inverse[symbol]) + ) + inverse[symbol] = label + self.first[name] = totalset + + def parse(self) -> tuple[dict[str, list["DFAState"]], str]: + dfas = {} + startsymbol: Optional[str] = None + # MSTART: (NEWLINE | RULE)* ENDMARKER + while self.type != token.ENDMARKER: + while self.type == token.NEWLINE: + self.gettoken() + # RULE: NAME ':' RHS NEWLINE + name = self.expect(token.NAME) + self.expect(token.OP, ":") + a, z = self.parse_rhs() + self.expect(token.NEWLINE) + # self.dump_nfa(name, a, z) + dfa = self.make_dfa(a, z) + # self.dump_dfa(name, dfa) + # oldlen = len(dfa) + self.simplify_dfa(dfa) + # newlen = len(dfa) + dfas[name] = dfa + # print name, oldlen, newlen + if startsymbol is None: + startsymbol = name + assert startsymbol is not None + return dfas, startsymbol + + def make_dfa(self, start: "NFAState", finish: "NFAState") -> list["DFAState"]: + # To turn an NFA into a DFA, we define the states of the DFA + # to correspond to *sets* of states of the NFA. Then do some + # state reduction. Let's represent sets as dicts with 1 for + # values. + assert isinstance(start, NFAState) + assert isinstance(finish, NFAState) + + def closure(state: NFAState) -> dict[NFAState, int]: + base: dict[NFAState, int] = {} + addclosure(state, base) + return base + + def addclosure(state: NFAState, base: dict[NFAState, int]) -> None: + assert isinstance(state, NFAState) + if state in base: + return + base[state] = 1 + for label, next in state.arcs: + if label is None: + addclosure(next, base) + + states = [DFAState(closure(start), finish)] + for state in states: # NB states grows while we're iterating + arcs: dict[str, dict[NFAState, int]] = {} + for nfastate in state.nfaset: + for label, next in nfastate.arcs: + if label is not None: + addclosure(next, arcs.setdefault(label, {})) + for label, nfaset in sorted(arcs.items()): + for st in states: + if st.nfaset == nfaset: + break + else: + st = DFAState(nfaset, finish) + states.append(st) + state.addarc(st, label) + return states # List of DFAState instances; first one is start + + def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None: + print("Dump of NFA for", name) + todo = [start] + for i, state in enumerate(todo): + print(" State", i, state is finish and "(final)" or "") + for label, next in state.arcs: + if next in todo: + j = todo.index(next) + else: + j = len(todo) + todo.append(next) + if label is None: + print(" -> %d" % j) + else: + print(" %s -> %d" % (label, j)) + + def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None: + print("Dump of DFA for", name) + for i, state in enumerate(dfa): + print(" State", i, state.isfinal and "(final)" or "") + for label, next in sorted(state.arcs.items()): + print(" %s -> %d" % (label, dfa.index(next))) + + def simplify_dfa(self, dfa: list["DFAState"]) -> None: + # This is not theoretically optimal, but works well enough. + # Algorithm: repeatedly look for two states that have the same + # set of arcs (same labels pointing to the same nodes) and + # unify them, until things stop changing. + + # dfa is a list of DFAState instances + changes = True + while changes: + changes = False + for i, state_i in enumerate(dfa): + for j in range(i + 1, len(dfa)): + state_j = dfa[j] + if state_i == state_j: + # print " unify", i, j + del dfa[j] + for state in dfa: + state.unifystate(state_j, state_i) + changes = True + break + + def parse_rhs(self) -> tuple["NFAState", "NFAState"]: + # RHS: ALT ('|' ALT)* + a, z = self.parse_alt() + if self.value != "|": + return a, z + else: + aa = NFAState() + zz = NFAState() + aa.addarc(a) + z.addarc(zz) + while self.value == "|": + self.gettoken() + a, z = self.parse_alt() + aa.addarc(a) + z.addarc(zz) + return aa, zz + + def parse_alt(self) -> tuple["NFAState", "NFAState"]: + # ALT: ITEM+ + a, b = self.parse_item() + while self.value in ("(", "[") or self.type in (token.NAME, token.STRING): + c, d = self.parse_item() + b.addarc(c) + b = d + return a, b + + def parse_item(self) -> tuple["NFAState", "NFAState"]: + # ITEM: '[' RHS ']' | ATOM ['+' | '*'] + if self.value == "[": + self.gettoken() + a, z = self.parse_rhs() + self.expect(token.OP, "]") + a.addarc(z) + return a, z + else: + a, z = self.parse_atom() + value = self.value + if value not in ("+", "*"): + return a, z + self.gettoken() + z.addarc(a) + if value == "+": + return a, z + else: + return a, a + + def parse_atom(self) -> tuple["NFAState", "NFAState"]: + # ATOM: '(' RHS ')' | NAME | STRING + if self.value == "(": + self.gettoken() + a, z = self.parse_rhs() + self.expect(token.OP, ")") + return a, z + elif self.type in (token.NAME, token.STRING): + a = NFAState() + z = NFAState() + a.addarc(z, self.value) + self.gettoken() + return a, z + else: + self.raise_error( + "expected (...) or NAME or STRING, got %s/%s", self.type, self.value + ) + raise AssertionError + + def expect(self, type: int, value: Optional[Any] = None) -> str: + if self.type != type or (value is not None and self.value != value): + self.raise_error( + "expected %s/%s, got %s/%s", type, value, self.type, self.value + ) + value = self.value + self.gettoken() + return value + + def gettoken(self) -> None: + tup = next(self.generator) + while tup[0] in (tokenize.COMMENT, tokenize.NL): + tup = next(self.generator) + self.type, self.value, self.begin, self.end, self.line = tup + # print token.tok_name[self.type], repr(self.value) + + def raise_error(self, msg: str, *args: Any) -> NoReturn: + if args: + try: + msg = msg % args + except Exception: + msg = " ".join([msg] + list(map(str, args))) + raise SyntaxError( + msg, (str(self.filename), self.end[0], self.end[1], self.line) + ) + + +class NFAState: + arcs: list[tuple[Optional[str], "NFAState"]] + + def __init__(self) -> None: + self.arcs = [] # list of (label, NFAState) pairs + + def addarc(self, next: "NFAState", label: Optional[str] = None) -> None: + assert label is None or isinstance(label, str) + assert isinstance(next, NFAState) + self.arcs.append((label, next)) + + +class DFAState: + nfaset: dict[NFAState, Any] + isfinal: bool + arcs: dict[str, "DFAState"] + + def __init__(self, nfaset: dict[NFAState, Any], final: NFAState) -> None: + assert isinstance(nfaset, dict) + assert isinstance(next(iter(nfaset)), NFAState) + assert isinstance(final, NFAState) + self.nfaset = nfaset + self.isfinal = final in nfaset + self.arcs = {} # map from label to DFAState + + def addarc(self, next: "DFAState", label: str) -> None: + assert isinstance(label, str) + assert label not in self.arcs + assert isinstance(next, DFAState) + self.arcs[label] = next + + def unifystate(self, old: "DFAState", new: "DFAState") -> None: + for label, next in self.arcs.items(): + if next is old: + self.arcs[label] = new + + def __eq__(self, other: Any) -> bool: + # Equality test -- ignore the nfaset instance variable + assert isinstance(other, DFAState) + if self.isfinal != other.isfinal: + return False + # Can't just return self.arcs == other.arcs, because that + # would invoke this method recursively, with cycles... + if len(self.arcs) != len(other.arcs): + return False + for label, next in self.arcs.items(): + if next is not other.arcs.get(label): + return False + return True + + __hash__: Any = None # For Py3 compatibility. + + +def generate_grammar(filename: Path = "Grammar.txt") -> PgenGrammar: + p = ParserGenerator(filename) + return p.make_grammar() diff --git a/contrib/python/black/blib2to3/pgen2/token.py b/contrib/python/black/blib2to3/pgen2/token.py new file mode 100644 index 00000000000..10c7c63bfe9 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/token.py @@ -0,0 +1,92 @@ +"""Token constants (from "token.h").""" + +from typing import Final + +# Taken from Python (r53757) and modified to include some tokens +# originally monkeypatched in by pgen2.tokenize + +# --start constants-- +ENDMARKER: Final = 0 +NAME: Final = 1 +NUMBER: Final = 2 +STRING: Final = 3 +NEWLINE: Final = 4 +INDENT: Final = 5 +DEDENT: Final = 6 +LPAR: Final = 7 +RPAR: Final = 8 +LSQB: Final = 9 +RSQB: Final = 10 +COLON: Final = 11 +COMMA: Final = 12 +SEMI: Final = 13 +PLUS: Final = 14 +MINUS: Final = 15 +STAR: Final = 16 +SLASH: Final = 17 +VBAR: Final = 18 +AMPER: Final = 19 +LESS: Final = 20 +GREATER: Final = 21 +EQUAL: Final = 22 +DOT: Final = 23 +PERCENT: Final = 24 +BACKQUOTE: Final = 25 +LBRACE: Final = 26 +RBRACE: Final = 27 +EQEQUAL: Final = 28 +NOTEQUAL: Final = 29 +LESSEQUAL: Final = 30 +GREATEREQUAL: Final = 31 +TILDE: Final = 32 +CIRCUMFLEX: Final = 33 +LEFTSHIFT: Final = 34 +RIGHTSHIFT: Final = 35 +DOUBLESTAR: Final = 36 +PLUSEQUAL: Final = 37 +MINEQUAL: Final = 38 +STAREQUAL: Final = 39 +SLASHEQUAL: Final = 40 +PERCENTEQUAL: Final = 41 +AMPEREQUAL: Final = 42 +VBAREQUAL: Final = 43 +CIRCUMFLEXEQUAL: Final = 44 +LEFTSHIFTEQUAL: Final = 45 +RIGHTSHIFTEQUAL: Final = 46 +DOUBLESTAREQUAL: Final = 47 +DOUBLESLASH: Final = 48 +DOUBLESLASHEQUAL: Final = 49 +AT: Final = 50 +ATEQUAL: Final = 51 +OP: Final = 52 +COMMENT: Final = 53 +NL: Final = 54 +RARROW: Final = 55 +AWAIT: Final = 56 +ASYNC: Final = 57 +ERRORTOKEN: Final = 58 +COLONEQUAL: Final = 59 +FSTRING_START: Final = 60 +FSTRING_MIDDLE: Final = 61 +FSTRING_END: Final = 62 +BANG: Final = 63 +N_TOKENS: Final = 64 +NT_OFFSET: Final = 256 +# --end constants-- + +tok_name: Final[dict[int, str]] = {} +for _name, _value in list(globals().items()): + if type(_value) is int: + tok_name[_value] = _name + + +def ISTERMINAL(x: int) -> bool: + return x < NT_OFFSET + + +def ISNONTERMINAL(x: int) -> bool: + return x >= NT_OFFSET + + +def ISEOF(x: int) -> bool: + return x == ENDMARKER diff --git a/contrib/python/black/blib2to3/pgen2/tokenize.py b/contrib/python/black/blib2to3/pgen2/tokenize.py new file mode 100644 index 00000000000..407c184dd74 --- /dev/null +++ b/contrib/python/black/blib2to3/pgen2/tokenize.py @@ -0,0 +1,1114 @@ +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. +# All rights reserved. + +# mypy: allow-untyped-defs, allow-untyped-calls + +"""Tokenization help for Python programs. + +generate_tokens(readline) is a generator that breaks a stream of +text into Python tokens. It accepts a readline-like method which is called +repeatedly to get the next line of input (or "" for EOF). It generates +5-tuples with these members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators + +Older entry points + tokenize_loop(readline, tokeneater) + tokenize(readline, tokeneater=printtoken) +are the same, except instead of generating tokens, tokeneater is a callback +function to which the 5 fields described above are passed as 5 arguments, +each time a new token is found.""" + +import builtins +import sys +from collections.abc import Callable, Iterable, Iterator +from re import Pattern +from typing import Final, Optional, Union + +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.token import ( + ASYNC, + AWAIT, + COMMENT, + DEDENT, + ENDMARKER, + ERRORTOKEN, + FSTRING_END, + FSTRING_MIDDLE, + FSTRING_START, + INDENT, + LBRACE, + NAME, + NEWLINE, + NL, + NUMBER, + OP, + RBRACE, + STRING, + tok_name, +) + +__author__ = "Ka-Ping Yee <[email protected]>" +__credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro" + +import re +from codecs import BOM_UTF8, lookup + +from . import token + +__all__ = [x for x in dir(token) if x[0] != "_"] + [ + "tokenize", + "generate_tokens", + "untokenize", +] +del token + + +def group(*choices: str) -> str: + return "(" + "|".join(choices) + ")" + + +def any(*choices: str) -> str: + return group(*choices) + "*" + + +def maybe(*choices: str) -> str: + return group(*choices) + "?" + + +def _combinations(*l: str) -> set[str]: + return {x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()} + + +Whitespace = r"[ \f\t]*" +Comment = r"#[^\r\n]*" +Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment) +Name = ( # this is invalid but it's fine because Name comes after Number in all groups + r"[^\s#\(\)\[\]\{\}+\-*/!@$%^&=|;:'\",\.<>/?`~\\]+" +) + +Binnumber = r"0[bB]_?[01]+(?:_[01]+)*" +Hexnumber = r"0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?" +Octnumber = r"0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?" +Decnumber = group(r"[1-9]\d*(?:_\d+)*[lL]?", "0[lL]?") +Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) +Exponent = r"[eE][-+]?\d+(?:_\d+)*" +Pointfloat = group(r"\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?", r"\.\d+(?:_\d+)*") + maybe( + Exponent +) +Expfloat = r"\d+(?:_\d+)*" + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r"\d+(?:_\d+)*[jJ]", Floatnumber + r"[jJ]") +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Tail end of ' string. +Single = r"(?:\\.|[^'\\])*'" +# Tail end of " string. +Double = r'(?:\\.|[^"\\])*"' +# Tail end of ''' string. +Single3 = r"(?:\\.|'(?!'')|[^'\\])*'''" +# Tail end of """ string. +Double3 = r'(?:\\.|"(?!"")|[^"\\])*"""' +_litprefix = r"(?:[uUrRbB]|[rR][bB]|[bBuU][rR])?" +_fstringlitprefix = r"(?:rF|FR|Fr|fr|RF|F|rf|f|Rf|fR)" +Triple = group( + _litprefix + "'''", + _litprefix + '"""', + _fstringlitprefix + '"""', + _fstringlitprefix + "'''", +) + +# beginning of a single quoted f-string. must not end with `{{` or `\N{` +SingleLbrace = r"(?:\\N{|{{|\\'|[^\n'{])*(?<!\\N)({)(?!{)" +DoubleLbrace = r'(?:\\N{|{{|\\"|[^\n"{])*(?<!\\N)({)(?!{)' + +# beginning of a triple quoted f-string. must not end with `{{` or `\N{` +Single3Lbrace = r"(?:\\N{|{{|\\'|'(?!'')|[^'{])*(?<!\\N){(?!{)" +Double3Lbrace = r'(?:\\N{|{{|\\"|"(?!"")|[^"{])*(?<!\\N){(?!{)' + +# ! format specifier inside an fstring brace, ensure it's not a `!=` token +Bang = Whitespace + group("!") + r"(?!=)" +bang = re.compile(Bang) +Colon = Whitespace + group(":") +colon = re.compile(Colon) + +FstringMiddleAfterColon = group(Whitespace + r".*?") + group("{", "}") +fstring_middle_after_colon = re.compile(FstringMiddleAfterColon) + +# Because of leftmost-then-longest match semantics, be sure to put the +# longest operators first (e.g., if = came before ==, == would get +# recognized as two instances of =). +Operator = group( + r"\*\*=?", + r">>=?", + r"<<=?", + r"<>", + r"!=", + r"//=?", + r"->", + r"[+\-*/%&@|^=<>:]=?", + r"~", +) + +Bracket = "[][(){}]" +Special = group(r"\r?\n", r"[:;.,`@]") +Funny = group(Operator, Bracket, Special) + +_string_middle_single = r"(?:[^\n'\\]|\\.)*" +_string_middle_double = r'(?:[^\n"\\]|\\.)*' + +# FSTRING_MIDDLE and LBRACE, must not end with a `{{` or `\N{` +_fstring_middle_single = SingleLbrace +_fstring_middle_double = DoubleLbrace + +# First (or only) line of ' or " string. +ContStr = group( + _litprefix + "'" + _string_middle_single + group("'", r"\\\r?\n"), + _litprefix + '"' + _string_middle_double + group('"', r"\\\r?\n"), + group(_fstringlitprefix + "'") + _fstring_middle_single, + group(_fstringlitprefix + '"') + _fstring_middle_double, + group(_fstringlitprefix + "'") + _string_middle_single + group("'", r"\\\r?\n"), + group(_fstringlitprefix + '"') + _string_middle_double + group('"', r"\\\r?\n"), +) +PseudoExtras = group(r"\\\r?\n", Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +pseudoprog: Final = re.compile(PseudoToken, re.UNICODE) + +singleprog = re.compile(Single) +singleprog_plus_lbrace = re.compile(group(SingleLbrace, Single)) +doubleprog = re.compile(Double) +doubleprog_plus_lbrace = re.compile(group(DoubleLbrace, Double)) + +single3prog = re.compile(Single3) +single3prog_plus_lbrace = re.compile(group(Single3Lbrace, Single3)) +double3prog = re.compile(Double3) +double3prog_plus_lbrace = re.compile(group(Double3Lbrace, Double3)) + +_strprefixes = _combinations("r", "R", "b", "B") | {"u", "U", "ur", "uR", "Ur", "UR"} +_fstring_prefixes = _combinations("r", "R", "f", "F") - {"r", "R"} + +endprogs: Final = { + "'": singleprog, + '"': doubleprog, + "'''": single3prog, + '"""': double3prog, + **{f"{prefix}'": singleprog for prefix in _strprefixes}, + **{f'{prefix}"': doubleprog for prefix in _strprefixes}, + **{f"{prefix}'": singleprog_plus_lbrace for prefix in _fstring_prefixes}, + **{f'{prefix}"': doubleprog_plus_lbrace for prefix in _fstring_prefixes}, + **{f"{prefix}'''": single3prog for prefix in _strprefixes}, + **{f'{prefix}"""': double3prog for prefix in _strprefixes}, + **{f"{prefix}'''": single3prog_plus_lbrace for prefix in _fstring_prefixes}, + **{f'{prefix}"""': double3prog_plus_lbrace for prefix in _fstring_prefixes}, +} + +triple_quoted: Final = ( + {"'''", '"""'} + | {f"{prefix}'''" for prefix in _strprefixes | _fstring_prefixes} + | {f'{prefix}"""' for prefix in _strprefixes | _fstring_prefixes} +) +single_quoted: Final = ( + {"'", '"'} + | {f"{prefix}'" for prefix in _strprefixes | _fstring_prefixes} + | {f'{prefix}"' for prefix in _strprefixes | _fstring_prefixes} +) +fstring_prefix: Final = tuple( + {f"{prefix}'" for prefix in _fstring_prefixes} + | {f'{prefix}"' for prefix in _fstring_prefixes} + | {f"{prefix}'''" for prefix in _fstring_prefixes} + | {f'{prefix}"""' for prefix in _fstring_prefixes} +) + +tabsize = 8 + + +class TokenError(Exception): + pass + + +class StopTokenizing(Exception): + pass + + +Coord = tuple[int, int] + + +def printtoken( + type: int, token: str, srow_col: Coord, erow_col: Coord, line: str +) -> None: # for testing + (srow, scol) = srow_col + (erow, ecol) = erow_col + print( + "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token)) + ) + + +TokenEater = Callable[[int, str, Coord, Coord, str], None] + + +def tokenize(readline: Callable[[], str], tokeneater: TokenEater = printtoken) -> None: + """ + The tokenize() function accepts two parameters: one representing the + input stream, and one providing an output mechanism for tokenize(). + + The first parameter, readline, must be a callable object which provides + the same interface as the readline() method of built-in file objects. + Each call to the function should return one line of input as a string. + + The second parameter, tokeneater, must also be a callable object. It is + called once for each token, with five arguments, corresponding to the + tuples generated by generate_tokens(). + """ + try: + tokenize_loop(readline, tokeneater) + except StopTokenizing: + pass + + +# backwards compatible interface +def tokenize_loop(readline: Callable[[], str], tokeneater: TokenEater) -> None: + for token_info in generate_tokens(readline): + tokeneater(*token_info) + + +GoodTokenInfo = tuple[int, str, Coord, Coord, str] +TokenInfo = Union[tuple[int, str], GoodTokenInfo] + + +class Untokenizer: + tokens: list[str] + prev_row: int + prev_col: int + + def __init__(self) -> None: + self.tokens = [] + self.prev_row = 1 + self.prev_col = 0 + + def add_whitespace(self, start: Coord) -> None: + row, col = start + assert row <= self.prev_row + col_offset = col - self.prev_col + if col_offset: + self.tokens.append(" " * col_offset) + + def untokenize(self, iterable: Iterable[TokenInfo]) -> str: + for t in iterable: + if len(t) == 2: + self.compat(t, iterable) + break + tok_type, token, start, end, line = t + self.add_whitespace(start) + self.tokens.append(token) + self.prev_row, self.prev_col = end + if tok_type in (NEWLINE, NL): + self.prev_row += 1 + self.prev_col = 0 + return "".join(self.tokens) + + def compat(self, token: tuple[int, str], iterable: Iterable[TokenInfo]) -> None: + startline = False + indents = [] + toks_append = self.tokens.append + toknum, tokval = token + if toknum in (NAME, NUMBER): + tokval += " " + if toknum in (NEWLINE, NL): + startline = True + for tok in iterable: + toknum, tokval = tok[:2] + + if toknum in (NAME, NUMBER, ASYNC, AWAIT): + tokval += " " + + if toknum == INDENT: + indents.append(tokval) + continue + elif toknum == DEDENT: + indents.pop() + continue + elif toknum in (NEWLINE, NL): + startline = True + elif startline and indents: + toks_append(indents[-1]) + startline = False + toks_append(tokval) + + +cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII) +blank_re = re.compile(rb"^[ \t\f]*(?:[#\r\n]|$)", re.ASCII) + + +def _get_normal_name(orig_enc: str) -> str: + """Imitates get_normal_name in tokenizer.c.""" + # Only care about the first 12 characters. + enc = orig_enc[:12].lower().replace("_", "-") + if enc == "utf-8" or enc.startswith("utf-8-"): + return "utf-8" + if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or enc.startswith( + ("latin-1-", "iso-8859-1-", "iso-latin-1-") + ): + return "iso-8859-1" + return orig_enc + + +def detect_encoding(readline: Callable[[], bytes]) -> tuple[str, list[bytes]]: + """ + The detect_encoding() function is used to detect the encoding that should + be used to decode a Python source file. It requires one argument, readline, + in the same way as the tokenize() generator. + + It will call readline a maximum of twice, and return the encoding used + (as a string) and a list of any lines (left as bytes) it has read + in. + + It detects the encoding from the presence of a utf-8 bom or an encoding + cookie as specified in pep-0263. If both a bom and a cookie are present, but + disagree, a SyntaxError will be raised. If the encoding cookie is an invalid + charset, raise a SyntaxError. Note that if a utf-8 bom is found, + 'utf-8-sig' is returned. + + If no encoding is specified, then the default of 'utf-8' will be returned. + """ + bom_found = False + encoding = None + default = "utf-8" + + def read_or_stop() -> bytes: + try: + return readline() + except StopIteration: + return b"" + + def find_cookie(line: bytes) -> Optional[str]: + try: + line_string = line.decode("ascii") + except UnicodeDecodeError: + return None + match = cookie_re.match(line_string) + if not match: + return None + encoding = _get_normal_name(match.group(1)) + try: + codec = lookup(encoding) + except LookupError: + # This behaviour mimics the Python interpreter + raise SyntaxError("unknown encoding: " + encoding) + + if bom_found: + if codec.name != "utf-8": + # This behaviour mimics the Python interpreter + raise SyntaxError("encoding problem: utf-8") + encoding += "-sig" + return encoding + + first = read_or_stop() + if first.startswith(BOM_UTF8): + bom_found = True + first = first[3:] + default = "utf-8-sig" + if not first: + return default, [] + + encoding = find_cookie(first) + if encoding: + return encoding, [first] + if not blank_re.match(first): + return default, [first] + + second = read_or_stop() + if not second: + return default, [first] + + encoding = find_cookie(second) + if encoding: + return encoding, [first, second] + + return default, [first, second] + + +def untokenize(iterable: Iterable[TokenInfo]) -> str: + """Transform tokens back into Python source code. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited input: + # Output text will tokenize the back to the input + t1 = [tok[:2] for tok in generate_tokens(f.readline)] + newcode = untokenize(t1) + readline = iter(newcode.splitlines(1)).next + t2 = [tok[:2] for tokin generate_tokens(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + return ut.untokenize(iterable) + + +def is_fstring_start(token: str) -> bool: + return token.startswith(fstring_prefix) + + +def _split_fstring_start_and_middle(token: str) -> tuple[str, str]: + for prefix in fstring_prefix: + _, prefix, rest = token.partition(prefix) + if prefix != "": + return prefix, rest + + raise ValueError(f"Token {token!r} is not a valid f-string start") + + +STATE_NOT_FSTRING: Final = 0 # not in an f-string +STATE_MIDDLE: Final = 1 # in the string portion of an f-string (outside braces) +STATE_IN_BRACES: Final = 2 # between braces in an f-string +# in the format specifier (between the colon and the closing brace) +STATE_IN_COLON: Final = 3 + + +class FStringState: + """Keeps track of state around f-strings. + + The tokenizer should call the appropriate method on this class when + it transitions to a different part of an f-string. This is needed + because the tokenization depends on knowing where exactly we are in + the f-string. + + For example, consider the following f-string: + + f"a{1:b{2}c}d" + + The following is the tokenization of this string and the states + tracked by this class: + + 1,0-1,2: FSTRING_START 'f"' # [STATE_NOT_FSTRING, STATE_MIDDLE] + 1,2-1,3: FSTRING_MIDDLE 'a' + 1,3-1,4: LBRACE '{' # [STATE_NOT_FSTRING, STATE_IN_BRACES] + 1,4-1,5: NUMBER '1' + 1,5-1,6: OP ':' # [STATE_NOT_FSTRING, STATE_IN_COLON] + 1,6-1,7: FSTRING_MIDDLE 'b' + 1,7-1,8: LBRACE '{' # [STATE_NOT_FSTRING, STATE_IN_COLON, STATE_IN_BRACES] + 1,8-1,9: NUMBER '2' + 1,9-1,10: RBRACE '}' # [STATE_NOT_FSTRING, STATE_IN_COLON] + 1,10-1,11: FSTRING_MIDDLE 'c' + 1,11-1,12: RBRACE '}' # [STATE_NOT_FSTRING, STATE_MIDDLE] + 1,12-1,13: FSTRING_MIDDLE 'd' + 1,13-1,14: FSTRING_END '"' # [STATE_NOT_FSTRING] + 1,14-1,15: NEWLINE '\n' + 2,0-2,0: ENDMARKER '' + + Notice that the nested braces in the format specifier are represented + by adding a STATE_IN_BRACES entry to the state stack. The stack is + also used if there are nested f-strings. + + """ + + def __init__(self) -> None: + self.stack: list[int] = [STATE_NOT_FSTRING] + + def is_in_fstring_expression(self) -> bool: + return self.stack[-1] not in (STATE_MIDDLE, STATE_NOT_FSTRING) + + def current(self) -> int: + return self.stack[-1] + + def enter_fstring(self) -> None: + self.stack.append(STATE_MIDDLE) + + def leave_fstring(self) -> None: + state = self.stack.pop() + assert state == STATE_MIDDLE + + def consume_lbrace(self) -> None: + current_state = self.stack[-1] + if current_state == STATE_MIDDLE: + self.stack[-1] = STATE_IN_BRACES + elif current_state == STATE_IN_COLON: + self.stack.append(STATE_IN_BRACES) + else: + assert False, current_state + + def consume_rbrace(self) -> None: + current_state = self.stack[-1] + assert current_state in (STATE_IN_BRACES, STATE_IN_COLON) + if len(self.stack) > 1 and self.stack[-2] == STATE_IN_COLON: + self.stack.pop() + else: + self.stack[-1] = STATE_MIDDLE + + def consume_colon(self) -> None: + assert self.stack[-1] == STATE_IN_BRACES, self.stack + self.stack[-1] = STATE_IN_COLON + + +def generate_tokens( + readline: Callable[[], str], grammar: Optional[Grammar] = None +) -> Iterator[GoodTokenInfo]: + """ + The generate_tokens() generator requires one argument, readline, which + must be a callable object which provides the same interface as the + readline() method of built-in file objects. Each call to the function + should return one line of input as a string. Alternately, readline + can be a callable function terminating with StopIteration: + readline = open(myfile).next # Example of alternate readline + + The generator produces 5-tuples with these members: the token type; the + token string; a 2-tuple (srow, scol) of ints specifying the row and + column where the token begins in the source; a 2-tuple (erow, ecol) of + ints specifying the row and column where the token ends in the source; + and the line on which the token was found. The line passed is the + logical line; continuation lines are included. + """ + lnum = parenlev = continued = 0 + parenlev_stack: list[int] = [] + fstring_state = FStringState() + formatspec = "" + numchars: Final[str] = "0123456789" + contstr, needcont = "", 0 + contline: Optional[str] = None + indents = [0] + + # If we know we're parsing 3.7+, we can unconditionally parse `async` and + # `await` as keywords. + async_keywords = False if grammar is None else grammar.async_keywords + # 'stashed' and 'async_*' are used for async/await parsing + stashed: Optional[GoodTokenInfo] = None + async_def = False + async_def_indent = 0 + async_def_nl = False + + strstart: tuple[int, int] + endprog_stack: list[Pattern[str]] = [] + formatspec_start: tuple[int, int] + + while 1: # loop over lines in stream + try: + line = readline() + except StopIteration: + line = "" + lnum += 1 + + # skip lines that are just indent characters ending with a slash + # to avoid storing that line's indent information. + if not contstr and line.rstrip("\n").strip(" \t\f") == "\\": + continue + + pos, max = 0, len(line) + + if contstr: # continued string + assert contline is not None + if not line: + raise TokenError("EOF in multi-line string", strstart) + endprog = endprog_stack[-1] + endmatch = endprog.match(line) + if endmatch: + end = endmatch.end(0) + token = contstr + line[:end] + spos = strstart + epos = (lnum, end) + tokenline = contline + line + if fstring_state.current() in ( + STATE_NOT_FSTRING, + STATE_IN_BRACES, + ) and not is_fstring_start(token): + yield (STRING, token, spos, epos, tokenline) + endprog_stack.pop() + parenlev = parenlev_stack.pop() + else: + if is_fstring_start(token): + fstring_start, token = _split_fstring_start_and_middle(token) + fstring_start_epos = (spos[0], spos[1] + len(fstring_start)) + yield ( + FSTRING_START, + fstring_start, + spos, + fstring_start_epos, + tokenline, + ) + fstring_state.enter_fstring() + # increase spos to the end of the fstring start + spos = fstring_start_epos + + if token.endswith("{"): + fstring_middle, lbrace = token[:-1], token[-1] + fstring_middle_epos = lbrace_spos = (lnum, end - 1) + yield ( + FSTRING_MIDDLE, + fstring_middle, + spos, + fstring_middle_epos, + line, + ) + yield (LBRACE, lbrace, lbrace_spos, epos, line) + fstring_state.consume_lbrace() + else: + if token.endswith(('"""', "'''")): + fstring_middle, fstring_end = token[:-3], token[-3:] + fstring_middle_epos = end_spos = (lnum, end - 3) + else: + fstring_middle, fstring_end = token[:-1], token[-1] + fstring_middle_epos = end_spos = (lnum, end - 1) + yield ( + FSTRING_MIDDLE, + fstring_middle, + spos, + fstring_middle_epos, + line, + ) + yield ( + FSTRING_END, + fstring_end, + end_spos, + epos, + line, + ) + fstring_state.leave_fstring() + endprog_stack.pop() + parenlev = parenlev_stack.pop() + pos = end + contstr, needcont = "", 0 + contline = None + elif needcont and line[-2:] != "\\\n" and line[-3:] != "\\\r\n": + yield ( + ERRORTOKEN, + contstr + line, + strstart, + (lnum, len(line)), + contline, + ) + contstr = "" + contline = None + continue + else: + contstr = contstr + line + contline = contline + line + continue + + # new statement + elif ( + parenlev == 0 + and not continued + and not fstring_state.is_in_fstring_expression() + ): + if not line: + break + column = 0 + while pos < max: # measure leading whitespace + if line[pos] == " ": + column += 1 + elif line[pos] == "\t": + column = (column // tabsize + 1) * tabsize + elif line[pos] == "\f": + column = 0 + else: + break + pos += 1 + if pos == max: + break + + if stashed: + yield stashed + stashed = None + + if line[pos] in "\r\n": # skip blank lines + yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line) + continue + + if line[pos] == "#": # skip comments + comment_token = line[pos:].rstrip("\r\n") + nl_pos = pos + len(comment_token) + yield ( + COMMENT, + comment_token, + (lnum, pos), + (lnum, nl_pos), + line, + ) + yield (NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line) + continue + + if column > indents[-1]: # count indents + indents.append(column) + yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + + while column < indents[-1]: # count dedents + if column not in indents: + raise IndentationError( + "unindent does not match any outer indentation level", + ("<tokenize>", lnum, pos, line), + ) + indents = indents[:-1] + + if async_def and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + + yield (DEDENT, "", (lnum, pos), (lnum, pos), line) + + if async_def and async_def_nl and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + + else: # continued statement + if not line: + raise TokenError("EOF in multi-line statement", (lnum, 0)) + continued = 0 + + while pos < max: + if fstring_state.current() == STATE_MIDDLE: + endprog = endprog_stack[-1] + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + start, end = endmatch.span(0) + token = line[start:end] + if token.endswith(('"""', "'''")): + middle_token, end_token = token[:-3], token[-3:] + middle_epos = end_spos = (lnum, end - 3) + else: + middle_token, end_token = token[:-1], token[-1] + middle_epos = end_spos = (lnum, end - 1) + # TODO: unsure if this can be safely removed + if stashed: + yield stashed + stashed = None + yield ( + FSTRING_MIDDLE, + middle_token, + (lnum, pos), + middle_epos, + line, + ) + if not token.endswith("{"): + yield ( + FSTRING_END, + end_token, + end_spos, + (lnum, end), + line, + ) + fstring_state.leave_fstring() + endprog_stack.pop() + parenlev = parenlev_stack.pop() + else: + yield (LBRACE, "{", (lnum, end - 1), (lnum, end), line) + fstring_state.consume_lbrace() + pos = end + continue + else: # multiple lines + strstart = (lnum, end) + contstr = line[end:] + contline = line + break + + if fstring_state.current() == STATE_IN_COLON: + match = fstring_middle_after_colon.match(line, pos) + if match is None: + formatspec += line[pos:] + pos = max + continue + + start, end = match.span(1) + token = line[start:end] + formatspec += token + + brace_start, brace_end = match.span(2) + brace_or_nl = line[brace_start:brace_end] + if brace_or_nl == "\n": + pos = brace_end + + yield (FSTRING_MIDDLE, formatspec, formatspec_start, (lnum, end), line) + formatspec = "" + + if brace_or_nl == "{": + yield (LBRACE, "{", (lnum, brace_start), (lnum, brace_end), line) + fstring_state.consume_lbrace() + end = brace_end + elif brace_or_nl == "}": + yield (RBRACE, "}", (lnum, brace_start), (lnum, brace_end), line) + fstring_state.consume_rbrace() + end = brace_end + formatspec_start = (lnum, brace_end) + + pos = end + continue + + if fstring_state.current() == STATE_IN_BRACES and parenlev == 0: + match = bang.match(line, pos) + if match: + start, end = match.span(1) + yield (OP, "!", (lnum, start), (lnum, end), line) + pos = end + continue + + match = colon.match(line, pos) + if match: + start, end = match.span(1) + yield (OP, ":", (lnum, start), (lnum, end), line) + fstring_state.consume_colon() + formatspec_start = (lnum, end) + pos = end + continue + + pseudomatch = pseudoprog.match(line, pos) + if pseudomatch: # scan for tokens + start, end = pseudomatch.span(1) + spos, epos, pos = (lnum, start), (lnum, end), end + token, initial = line[start:end], line[start] + + if initial in numchars or ( + initial == "." and token != "." + ): # ordinary number + yield (NUMBER, token, spos, epos, line) + elif initial in "\r\n": + newline = NEWLINE + if parenlev > 0 or fstring_state.is_in_fstring_expression(): + newline = NL + elif async_def: + async_def_nl = True + if stashed: + yield stashed + stashed = None + yield (newline, token, spos, epos, line) + + elif initial == "#": + assert not token.endswith("\n") + if stashed: + yield stashed + stashed = None + yield (COMMENT, token, spos, epos, line) + elif token in triple_quoted: + endprog = endprogs[token] + endprog_stack.append(endprog) + parenlev_stack.append(parenlev) + parenlev = 0 + if is_fstring_start(token): + yield (FSTRING_START, token, spos, epos, line) + fstring_state.enter_fstring() + + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + if stashed: + yield stashed + stashed = None + if not is_fstring_start(token): + pos = endmatch.end(0) + token = line[start:pos] + epos = (lnum, pos) + yield (STRING, token, spos, epos, line) + endprog_stack.pop() + parenlev = parenlev_stack.pop() + else: + end = endmatch.end(0) + token = line[pos:end] + spos, epos = (lnum, pos), (lnum, end) + if not token.endswith("{"): + fstring_middle, fstring_end = token[:-3], token[-3:] + fstring_middle_epos = fstring_end_spos = (lnum, end - 3) + yield ( + FSTRING_MIDDLE, + fstring_middle, + spos, + fstring_middle_epos, + line, + ) + yield ( + FSTRING_END, + fstring_end, + fstring_end_spos, + epos, + line, + ) + fstring_state.leave_fstring() + endprog_stack.pop() + parenlev = parenlev_stack.pop() + else: + fstring_middle, lbrace = token[:-1], token[-1] + fstring_middle_epos = lbrace_spos = (lnum, end - 1) + yield ( + FSTRING_MIDDLE, + fstring_middle, + spos, + fstring_middle_epos, + line, + ) + yield (LBRACE, lbrace, lbrace_spos, epos, line) + fstring_state.consume_lbrace() + pos = end + else: + # multiple lines + if is_fstring_start(token): + strstart = (lnum, pos) + contstr = line[pos:] + else: + strstart = (lnum, start) + contstr = line[start:] + contline = line + break + elif ( + initial in single_quoted + or token[:2] in single_quoted + or token[:3] in single_quoted + ): + maybe_endprog = ( + endprogs.get(initial) + or endprogs.get(token[:2]) + or endprogs.get(token[:3]) + ) + assert maybe_endprog is not None, f"endprog not found for {token}" + endprog = maybe_endprog + if token[-1] == "\n": # continued string + endprog_stack.append(endprog) + parenlev_stack.append(parenlev) + parenlev = 0 + strstart = (lnum, start) + contstr, needcont = line[start:], 1 + contline = line + break + else: # ordinary string + if stashed: + yield stashed + stashed = None + + if not is_fstring_start(token): + yield (STRING, token, spos, epos, line) + else: + if pseudomatch[20] is not None: + fstring_start = pseudomatch[20] + offset = pseudomatch.end(20) - pseudomatch.start(1) + elif pseudomatch[22] is not None: + fstring_start = pseudomatch[22] + offset = pseudomatch.end(22) - pseudomatch.start(1) + elif pseudomatch[24] is not None: + fstring_start = pseudomatch[24] + offset = pseudomatch.end(24) - pseudomatch.start(1) + else: + fstring_start = pseudomatch[26] + offset = pseudomatch.end(26) - pseudomatch.start(1) + + start_epos = (lnum, start + offset) + yield (FSTRING_START, fstring_start, spos, start_epos, line) + fstring_state.enter_fstring() + endprog = endprogs[fstring_start] + endprog_stack.append(endprog) + parenlev_stack.append(parenlev) + parenlev = 0 + + end_offset = pseudomatch.end(1) - 1 + fstring_middle = line[start + offset : end_offset] + middle_spos = (lnum, start + offset) + middle_epos = (lnum, end_offset) + yield ( + FSTRING_MIDDLE, + fstring_middle, + middle_spos, + middle_epos, + line, + ) + if not token.endswith("{"): + end_spos = (lnum, end_offset) + end_epos = (lnum, end_offset + 1) + yield (FSTRING_END, token[-1], end_spos, end_epos, line) + fstring_state.leave_fstring() + endprog_stack.pop() + parenlev = parenlev_stack.pop() + else: + end_spos = (lnum, end_offset) + end_epos = (lnum, end_offset + 1) + yield (LBRACE, "{", end_spos, end_epos, line) + fstring_state.consume_lbrace() + + elif initial.isidentifier(): # ordinary name + if token in ("async", "await"): + if async_keywords or async_def: + yield ( + ASYNC if token == "async" else AWAIT, + token, + spos, + epos, + line, + ) + continue + + tok = (NAME, token, spos, epos, line) + if token == "async" and not stashed: + stashed = tok + continue + + if token in ("def", "for"): + if stashed and stashed[0] == NAME and stashed[1] == "async": + if token == "def": + async_def = True + async_def_indent = indents[-1] + + yield ( + ASYNC, + stashed[1], + stashed[2], + stashed[3], + stashed[4], + ) + stashed = None + + if stashed: + yield stashed + stashed = None + + yield tok + elif initial == "\\": # continued stmt + # This yield is new; needed for better idempotency: + if stashed: + yield stashed + stashed = None + yield (NL, token, spos, (lnum, pos), line) + continued = 1 + elif ( + initial == "}" + and parenlev == 0 + and fstring_state.is_in_fstring_expression() + ): + yield (RBRACE, token, spos, epos, line) + fstring_state.consume_rbrace() + formatspec_start = epos + else: + if initial in "([{": + parenlev += 1 + elif initial in ")]}": + parenlev -= 1 + if stashed: + yield stashed + stashed = None + yield (OP, token, spos, epos, line) + else: + yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line) + pos += 1 + + if stashed: + yield stashed + stashed = None + + for _indent in indents[1:]: # pop remaining indent levels + yield (DEDENT, "", (lnum, 0), (lnum, 0), "") + yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "") + assert len(endprog_stack) == 0 + assert len(parenlev_stack) == 0 + + +if __name__ == "__main__": # testing + if len(sys.argv) > 1: + tokenize(open(sys.argv[1]).readline) + else: + tokenize(sys.stdin.readline) diff --git a/contrib/python/black/blib2to3/pygram.py b/contrib/python/black/blib2to3/pygram.py new file mode 100644 index 00000000000..70a5684bb07 --- /dev/null +++ b/contrib/python/black/blib2to3/pygram.py @@ -0,0 +1,204 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Export the Python grammar and symbols.""" + +# Python imports +import os +from typing import Union + +# Local imports +from .pgen2 import driver +from .pgen2.grammar import Grammar + +# Moved into initialize because mypyc can't handle __file__ (XXX bug) +# # The grammar file +# _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") +# _PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), +# "PatternGrammar.txt") + + +class Symbols: + def __init__(self, grammar: Grammar) -> None: + """Initializer. + + Creates an attribute for each grammar symbol (nonterminal), + whose value is the symbol's type (an int >= 256). + """ + for name, symbol in grammar.symbol2number.items(): + setattr(self, name, symbol) + + +class _python_symbols(Symbols): + and_expr: int + and_test: int + annassign: int + arglist: int + argument: int + arith_expr: int + asexpr_test: int + assert_stmt: int + async_funcdef: int + async_stmt: int + atom: int + augassign: int + break_stmt: int + case_block: int + classdef: int + comp_for: int + comp_if: int + comp_iter: int + comp_op: int + comparison: int + compound_stmt: int + continue_stmt: int + decorated: int + decorator: int + decorators: int + del_stmt: int + dictsetmaker: int + dotted_as_name: int + dotted_as_names: int + dotted_name: int + encoding_decl: int + eval_input: int + except_clause: int + expr: int + expr_stmt: int + exprlist: int + factor: int + file_input: int + flow_stmt: int + for_stmt: int + fstring: int + fstring_format_spec: int + fstring_middle: int + fstring_replacement_field: int + funcdef: int + global_stmt: int + guard: int + if_stmt: int + import_as_name: int + import_as_names: int + import_from: int + import_name: int + import_stmt: int + lambdef: int + listmaker: int + match_stmt: int + namedexpr_test: int + not_test: int + old_comp_for: int + old_comp_if: int + old_comp_iter: int + old_lambdef: int + old_test: int + or_test: int + parameters: int + paramspec: int + pass_stmt: int + pattern: int + patterns: int + power: int + raise_stmt: int + return_stmt: int + shift_expr: int + simple_stmt: int + single_input: int + sliceop: int + small_stmt: int + subject_expr: int + star_expr: int + stmt: int + subscript: int + subscriptlist: int + suite: int + term: int + test: int + testlist: int + testlist1: int + testlist_gexp: int + testlist_safe: int + testlist_star_expr: int + tfpdef: int + tfplist: int + tname: int + tname_star: int + trailer: int + try_stmt: int + type_stmt: int + typedargslist: int + typeparam: int + typeparams: int + typevar: int + typevartuple: int + varargslist: int + vfpdef: int + vfplist: int + vname: int + while_stmt: int + with_stmt: int + xor_expr: int + yield_arg: int + yield_expr: int + yield_stmt: int + + +class _pattern_symbols(Symbols): + Alternative: int + Alternatives: int + Details: int + Matcher: int + NegatedUnit: int + Repeater: int + Unit: int + + +python_grammar: Grammar +python_grammar_async_keywords: Grammar +python_grammar_soft_keywords: Grammar +pattern_grammar: Grammar +python_symbols: _python_symbols +pattern_symbols: _pattern_symbols + + +def initialize(cache_dir: Union[str, "os.PathLike[str]", None] = None) -> None: + global python_grammar + global python_grammar_async_keywords + global python_grammar_soft_keywords + global python_symbols + global pattern_grammar + global pattern_symbols + + # The grammar file + _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") + _PATTERN_GRAMMAR_FILE = os.path.join( + os.path.dirname(__file__), "PatternGrammar.txt" + ) + + python_grammar = driver.load_packaged_grammar("blib2to3", _GRAMMAR_FILE, cache_dir) + assert "print" not in python_grammar.keywords + assert "exec" not in python_grammar.keywords + + soft_keywords = python_grammar.soft_keywords.copy() + python_grammar.soft_keywords.clear() + + python_symbols = _python_symbols(python_grammar) + + # Python 3.0-3.6 + python_grammar.version = (3, 0) + + # Python 3.7+ + python_grammar_async_keywords = python_grammar.copy() + python_grammar_async_keywords.async_keywords = True + python_grammar_async_keywords.version = (3, 7) + + # Python 3.10+ + python_grammar_soft_keywords = python_grammar_async_keywords.copy() + python_grammar_soft_keywords.soft_keywords = soft_keywords + python_grammar_soft_keywords.version = (3, 10) + + pattern_grammar = driver.load_packaged_grammar( + "blib2to3", _PATTERN_GRAMMAR_FILE, cache_dir + ) + pattern_symbols = _pattern_symbols(pattern_grammar) diff --git a/contrib/python/black/blib2to3/pytree.py b/contrib/python/black/blib2to3/pytree.py new file mode 100644 index 00000000000..d57584685a2 --- /dev/null +++ b/contrib/python/black/blib2to3/pytree.py @@ -0,0 +1,976 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +""" +Python parse tree definitions. + +This is a very concrete parse tree; we need to keep every token and +even the comments and whitespace between tokens. + +There's also a pattern matching implementation here. +""" + +# mypy: allow-untyped-defs, allow-incomplete-defs + +from collections.abc import Iterable, Iterator +from typing import Any, Optional, TypeVar, Union + +from blib2to3.pgen2.grammar import Grammar + +__author__ = "Guido van Rossum <[email protected]>" + +import sys +from io import StringIO + +HUGE: int = 0x7FFFFFFF # maximum repeat count, default max + +_type_reprs: dict[int, Union[str, int]] = {} + + +def type_repr(type_num: int) -> Union[str, int]: + global _type_reprs + if not _type_reprs: + from . import pygram + + if not hasattr(pygram, "python_symbols"): + pygram.initialize(cache_dir=None) + + # printing tokens is possible but not as useful + # from .pgen2 import token // token.__dict__.items(): + for name in dir(pygram.python_symbols): + val = getattr(pygram.python_symbols, name) + if type(val) == int: + _type_reprs[val] = name + return _type_reprs.setdefault(type_num, type_num) + + +_P = TypeVar("_P", bound="Base") + +NL = Union["Node", "Leaf"] +Context = tuple[str, tuple[int, int]] +RawNode = tuple[int, Optional[str], Optional[Context], Optional[list[NL]]] + + +class Base: + """ + Abstract base class for Node and Leaf. + + This provides some default functionality and boilerplate using the + template pattern. + + A node may be a subnode of at most one parent. + """ + + # Default values for instance variables + type: int # int: token number (< 256) or symbol number (>= 256) + parent: Optional["Node"] = None # Parent node pointer, or None + children: list[NL] # List of subnodes + was_changed: bool = False + was_checked: bool = False + + def __new__(cls, *args, **kwds): + """Constructor that prevents Base from being instantiated.""" + assert cls is not Base, "Cannot instantiate Base" + return object.__new__(cls) + + def __eq__(self, other: Any) -> bool: + """ + Compare two nodes for equality. + + This calls the method _eq(). + """ + if self.__class__ is not other.__class__: + return NotImplemented + return self._eq(other) + + @property + def prefix(self) -> str: + raise NotImplementedError + + def _eq(self: _P, other: _P) -> bool: + """ + Compare two nodes for equality. + + This is called by __eq__ and __ne__. It is only called if the two nodes + have the same type. This must be implemented by the concrete subclass. + Nodes should be considered equal if they have the same structure, + ignoring the prefix string and other context information. + """ + raise NotImplementedError + + def __deepcopy__(self: _P, memo: Any) -> _P: + return self.clone() + + def clone(self: _P) -> _P: + """ + Return a cloned (deep) copy of self. + + This must be implemented by the concrete subclass. + """ + raise NotImplementedError + + def post_order(self) -> Iterator[NL]: + """ + Return a post-order iterator for the tree. + + This must be implemented by the concrete subclass. + """ + raise NotImplementedError + + def pre_order(self) -> Iterator[NL]: + """ + Return a pre-order iterator for the tree. + + This must be implemented by the concrete subclass. + """ + raise NotImplementedError + + def replace(self, new: Union[NL, list[NL]]) -> None: + """Replace this node with a new one in the parent.""" + assert self.parent is not None, str(self) + assert new is not None + if not isinstance(new, list): + new = [new] + l_children = [] + found = False + for ch in self.parent.children: + if ch is self: + assert not found, (self.parent.children, self, new) + if new is not None: + l_children.extend(new) + found = True + else: + l_children.append(ch) + assert found, (self.children, self, new) + self.parent.children = l_children + self.parent.changed() + self.parent.invalidate_sibling_maps() + for x in new: + x.parent = self.parent + self.parent = None + + def get_lineno(self) -> Optional[int]: + """Return the line number which generated the invocant node.""" + node = self + while not isinstance(node, Leaf): + if not node.children: + return None + node = node.children[0] + return node.lineno + + def changed(self) -> None: + if self.was_changed: + return + if self.parent: + self.parent.changed() + self.was_changed = True + + def remove(self) -> Optional[int]: + """ + Remove the node from the tree. Returns the position of the node in its + parent's children before it was removed. + """ + if self.parent: + for i, node in enumerate(self.parent.children): + if node is self: + del self.parent.children[i] + self.parent.changed() + self.parent.invalidate_sibling_maps() + self.parent = None + return i + return None + + @property + def next_sibling(self) -> Optional[NL]: + """ + The node immediately following the invocant in their parent's children + list. If the invocant does not have a next sibling, it is None + """ + if self.parent is None: + return None + + if self.parent.next_sibling_map is None: + self.parent.update_sibling_maps() + assert self.parent.next_sibling_map is not None + return self.parent.next_sibling_map[id(self)] + + @property + def prev_sibling(self) -> Optional[NL]: + """ + The node immediately preceding the invocant in their parent's children + list. If the invocant does not have a previous sibling, it is None. + """ + if self.parent is None: + return None + + if self.parent.prev_sibling_map is None: + self.parent.update_sibling_maps() + assert self.parent.prev_sibling_map is not None + return self.parent.prev_sibling_map[id(self)] + + def leaves(self) -> Iterator["Leaf"]: + for child in self.children: + yield from child.leaves() + + def depth(self) -> int: + if self.parent is None: + return 0 + return 1 + self.parent.depth() + + def get_suffix(self) -> str: + """ + Return the string immediately following the invocant node. This is + effectively equivalent to node.next_sibling.prefix + """ + next_sib = self.next_sibling + if next_sib is None: + return "" + prefix = next_sib.prefix + return prefix + + +class Node(Base): + """Concrete implementation for interior nodes.""" + + fixers_applied: Optional[list[Any]] + used_names: Optional[set[str]] + + def __init__( + self, + type: int, + children: list[NL], + context: Optional[Any] = None, + prefix: Optional[str] = None, + fixers_applied: Optional[list[Any]] = None, + ) -> None: + """ + Initializer. + + Takes a type constant (a symbol number >= 256), a sequence of + child nodes, and an optional context keyword argument. + + As a side effect, the parent pointers of the children are updated. + """ + assert type >= 256, type + self.type = type + self.children = list(children) + for ch in self.children: + assert ch.parent is None, repr(ch) + ch.parent = self + self.invalidate_sibling_maps() + if prefix is not None: + self.prefix = prefix + if fixers_applied: + self.fixers_applied = fixers_applied[:] + else: + self.fixers_applied = None + + def __repr__(self) -> str: + """Return a canonical string representation.""" + assert self.type is not None + return "{}({}, {!r})".format( + self.__class__.__name__, + type_repr(self.type), + self.children, + ) + + def __str__(self) -> str: + """ + Return a pretty string representation. + + This reproduces the input source exactly. + """ + return "".join(map(str, self.children)) + + def _eq(self, other: Base) -> bool: + """Compare two nodes for equality.""" + return (self.type, self.children) == (other.type, other.children) + + def clone(self) -> "Node": + assert self.type is not None + """Return a cloned (deep) copy of self.""" + return Node( + self.type, + [ch.clone() for ch in self.children], + fixers_applied=self.fixers_applied, + ) + + def post_order(self) -> Iterator[NL]: + """Return a post-order iterator for the tree.""" + for child in self.children: + yield from child.post_order() + yield self + + def pre_order(self) -> Iterator[NL]: + """Return a pre-order iterator for the tree.""" + yield self + for child in self.children: + yield from child.pre_order() + + @property + def prefix(self) -> str: + """ + The whitespace and comments preceding this node in the input. + """ + if not self.children: + return "" + return self.children[0].prefix + + @prefix.setter + def prefix(self, prefix: str) -> None: + if self.children: + self.children[0].prefix = prefix + + def set_child(self, i: int, child: NL) -> None: + """ + Equivalent to 'node.children[i] = child'. This method also sets the + child's parent attribute appropriately. + """ + child.parent = self + self.children[i].parent = None + self.children[i] = child + self.changed() + self.invalidate_sibling_maps() + + def insert_child(self, i: int, child: NL) -> None: + """ + Equivalent to 'node.children.insert(i, child)'. This method also sets + the child's parent attribute appropriately. + """ + child.parent = self + self.children.insert(i, child) + self.changed() + self.invalidate_sibling_maps() + + def append_child(self, child: NL) -> None: + """ + Equivalent to 'node.children.append(child)'. This method also sets the + child's parent attribute appropriately. + """ + child.parent = self + self.children.append(child) + self.changed() + self.invalidate_sibling_maps() + + def invalidate_sibling_maps(self) -> None: + self.prev_sibling_map: Optional[dict[int, Optional[NL]]] = None + self.next_sibling_map: Optional[dict[int, Optional[NL]]] = None + + def update_sibling_maps(self) -> None: + _prev: dict[int, Optional[NL]] = {} + _next: dict[int, Optional[NL]] = {} + self.prev_sibling_map = _prev + self.next_sibling_map = _next + previous: Optional[NL] = None + for current in self.children: + _prev[id(current)] = previous + _next[id(previous)] = current + previous = current + _next[id(current)] = None + + +class Leaf(Base): + """Concrete implementation for leaf nodes.""" + + # Default values for instance variables + value: str + fixers_applied: list[Any] + bracket_depth: int + # Changed later in brackets.py + opening_bracket: Optional["Leaf"] = None + used_names: Optional[set[str]] + _prefix = "" # Whitespace and comments preceding this token in the input + lineno: int = 0 # Line where this token starts in the input + column: int = 0 # Column where this token starts in the input + # If not None, this Leaf is created by converting a block of fmt off/skip + # code, and `fmt_pass_converted_first_leaf` points to the first Leaf in the + # converted code. + fmt_pass_converted_first_leaf: Optional["Leaf"] = None + + def __init__( + self, + type: int, + value: str, + context: Optional[Context] = None, + prefix: Optional[str] = None, + fixers_applied: list[Any] = [], + opening_bracket: Optional["Leaf"] = None, + fmt_pass_converted_first_leaf: Optional["Leaf"] = None, + ) -> None: + """ + Initializer. + + Takes a type constant (a token number < 256), a string value, and an + optional context keyword argument. + """ + + assert 0 <= type < 256, type + if context is not None: + self._prefix, (self.lineno, self.column) = context + self.type = type + self.value = value + if prefix is not None: + self._prefix = prefix + self.fixers_applied: Optional[list[Any]] = fixers_applied[:] + self.children = [] + self.opening_bracket = opening_bracket + self.fmt_pass_converted_first_leaf = fmt_pass_converted_first_leaf + + def __repr__(self) -> str: + """Return a canonical string representation.""" + from .pgen2.token import tok_name + + assert self.type is not None + return "{}({}, {!r})".format( + self.__class__.__name__, + tok_name.get(self.type, self.type), + self.value, + ) + + def __str__(self) -> str: + """ + Return a pretty string representation. + + This reproduces the input source exactly. + """ + return self._prefix + str(self.value) + + def _eq(self, other: "Leaf") -> bool: + """Compare two nodes for equality.""" + return (self.type, self.value) == (other.type, other.value) + + def clone(self) -> "Leaf": + assert self.type is not None + """Return a cloned (deep) copy of self.""" + return Leaf( + self.type, + self.value, + (self.prefix, (self.lineno, self.column)), + fixers_applied=self.fixers_applied, + ) + + def leaves(self) -> Iterator["Leaf"]: + yield self + + def post_order(self) -> Iterator["Leaf"]: + """Return a post-order iterator for the tree.""" + yield self + + def pre_order(self) -> Iterator["Leaf"]: + """Return a pre-order iterator for the tree.""" + yield self + + @property + def prefix(self) -> str: + """ + The whitespace and comments preceding this token in the input. + """ + return self._prefix + + @prefix.setter + def prefix(self, prefix: str) -> None: + self.changed() + self._prefix = prefix + + +def convert(gr: Grammar, raw_node: RawNode) -> NL: + """ + Convert raw node information to a Node or Leaf instance. + + This is passed to the parser driver which calls it whenever a reduction of a + grammar rule produces a new complete node, so that the tree is build + strictly bottom-up. + """ + type, value, context, children = raw_node + if children or type in gr.number2symbol: + # If there's exactly one child, return that child instead of + # creating a new node. + assert children is not None + if len(children) == 1: + return children[0] + return Node(type, children, context=context) + else: + return Leaf(type, value or "", context=context) + + +_Results = dict[str, NL] + + +class BasePattern: + """ + A pattern is a tree matching pattern. + + It looks for a specific node type (token or symbol), and + optionally for a specific content. + + This is an abstract base class. There are three concrete + subclasses: + + - LeafPattern matches a single leaf node; + - NodePattern matches a single node (usually non-leaf); + - WildcardPattern matches a sequence of nodes of variable length. + """ + + # Defaults for instance variables + type: Optional[int] + type = None # Node type (token if < 256, symbol if >= 256) + content: Any = None # Optional content matching pattern + name: Optional[str] = None # Optional name used to store match in results dict + + def __new__(cls, *args, **kwds): + """Constructor that prevents BasePattern from being instantiated.""" + assert cls is not BasePattern, "Cannot instantiate BasePattern" + return object.__new__(cls) + + def __repr__(self) -> str: + assert self.type is not None + args = [type_repr(self.type), self.content, self.name] + while args and args[-1] is None: + del args[-1] + return "{}({})".format(self.__class__.__name__, ", ".join(map(repr, args))) + + def _submatch(self, node, results=None) -> bool: + raise NotImplementedError + + def optimize(self) -> "BasePattern": + """ + A subclass can define this as a hook for optimizations. + + Returns either self or another node with the same effect. + """ + return self + + def match(self, node: NL, results: Optional[_Results] = None) -> bool: + """ + Does this pattern exactly match a node? + + Returns True if it matches, False if not. + + If results is not None, it must be a dict which will be + updated with the nodes matching named subpatterns. + + Default implementation for non-wildcard patterns. + """ + if self.type is not None and node.type != self.type: + return False + if self.content is not None: + r: Optional[_Results] = None + if results is not None: + r = {} + if not self._submatch(node, r): + return False + if r: + assert results is not None + results.update(r) + if results is not None and self.name: + results[self.name] = node + return True + + def match_seq(self, nodes: list[NL], results: Optional[_Results] = None) -> bool: + """ + Does this pattern exactly match a sequence of nodes? + + Default implementation for non-wildcard patterns. + """ + if len(nodes) != 1: + return False + return self.match(nodes[0], results) + + def generate_matches(self, nodes: list[NL]) -> Iterator[tuple[int, _Results]]: + """ + Generator yielding all matches for this pattern. + + Default implementation for non-wildcard patterns. + """ + r: _Results = {} + if nodes and self.match(nodes[0], r): + yield 1, r + + +class LeafPattern(BasePattern): + def __init__( + self, + type: Optional[int] = None, + content: Optional[str] = None, + name: Optional[str] = None, + ) -> None: + """ + Initializer. Takes optional type, content, and name. + + The type, if given must be a token type (< 256). If not given, + this matches any *leaf* node; the content may still be required. + + The content, if given, must be a string. + + If a name is given, the matching node is stored in the results + dict under that key. + """ + if type is not None: + assert 0 <= type < 256, type + if content is not None: + assert isinstance(content, str), repr(content) + self.type = type + self.content = content + self.name = name + + def match(self, node: NL, results=None) -> bool: + """Override match() to insist on a leaf node.""" + if not isinstance(node, Leaf): + return False + return BasePattern.match(self, node, results) + + def _submatch(self, node, results=None): + """ + Match the pattern's content to the node's children. + + This assumes the node type matches and self.content is not None. + + Returns True if it matches, False if not. + + If results is not None, it must be a dict which will be + updated with the nodes matching named subpatterns. + + When returning False, the results dict may still be updated. + """ + return self.content == node.value + + +class NodePattern(BasePattern): + wildcards: bool = False + + def __init__( + self, + type: Optional[int] = None, + content: Optional[Iterable[str]] = None, + name: Optional[str] = None, + ) -> None: + """ + Initializer. Takes optional type, content, and name. + + The type, if given, must be a symbol type (>= 256). If the + type is None this matches *any* single node (leaf or not), + except if content is not None, in which it only matches + non-leaf nodes that also match the content pattern. + + The content, if not None, must be a sequence of Patterns that + must match the node's children exactly. If the content is + given, the type must not be None. + + If a name is given, the matching node is stored in the results + dict under that key. + """ + if type is not None: + assert type >= 256, type + if content is not None: + assert not isinstance(content, str), repr(content) + newcontent = list(content) + for i, item in enumerate(newcontent): + assert isinstance(item, BasePattern), (i, item) + # I don't even think this code is used anywhere, but it does cause + # unreachable errors from mypy. This function's signature does look + # odd though *shrug*. + if isinstance(item, WildcardPattern): # type: ignore[unreachable] + self.wildcards = True # type: ignore[unreachable] + self.type = type + self.content = newcontent # TODO: this is unbound when content is None + self.name = name + + def _submatch(self, node, results=None) -> bool: + """ + Match the pattern's content to the node's children. + + This assumes the node type matches and self.content is not None. + + Returns True if it matches, False if not. + + If results is not None, it must be a dict which will be + updated with the nodes matching named subpatterns. + + When returning False, the results dict may still be updated. + """ + if self.wildcards: + for c, r in generate_matches(self.content, node.children): + if c == len(node.children): + if results is not None: + results.update(r) + return True + return False + if len(self.content) != len(node.children): + return False + for subpattern, child in zip(self.content, node.children): + if not subpattern.match(child, results): + return False + return True + + +class WildcardPattern(BasePattern): + """ + A wildcard pattern can match zero or more nodes. + + This has all the flexibility needed to implement patterns like: + + .* .+ .? .{m,n} + (a b c | d e | f) + (...)* (...)+ (...)? (...){m,n} + + except it always uses non-greedy matching. + """ + + min: int + max: int + + def __init__( + self, + content: Optional[str] = None, + min: int = 0, + max: int = HUGE, + name: Optional[str] = None, + ) -> None: + """ + Initializer. + + Args: + content: optional sequence of subsequences of patterns; + if absent, matches one node; + if present, each subsequence is an alternative [*] + min: optional minimum number of times to match, default 0 + max: optional maximum number of times to match, default HUGE + name: optional name assigned to this match + + [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is + equivalent to (a b c | d e | f g h); if content is None, + this is equivalent to '.' in regular expression terms. + The min and max parameters work as follows: + min=0, max=maxint: .* + min=1, max=maxint: .+ + min=0, max=1: .? + min=1, max=1: . + If content is not None, replace the dot with the parenthesized + list of alternatives, e.g. (a b c | d e | f g h)* + """ + assert 0 <= min <= max <= HUGE, (min, max) + if content is not None: + f = lambda s: tuple(s) + wrapped_content = tuple(map(f, content)) # Protect against alterations + # Check sanity of alternatives + assert len(wrapped_content), repr( + wrapped_content + ) # Can't have zero alternatives + for alt in wrapped_content: + assert len(alt), repr(alt) # Can have empty alternatives + self.content = wrapped_content + self.min = min + self.max = max + self.name = name + + def optimize(self) -> Any: + """Optimize certain stacked wildcard patterns.""" + subpattern = None + if ( + self.content is not None + and len(self.content) == 1 + and len(self.content[0]) == 1 + ): + subpattern = self.content[0][0] + if self.min == 1 and self.max == 1: + if self.content is None: + return NodePattern(name=self.name) + if subpattern is not None and self.name == subpattern.name: + return subpattern.optimize() + if ( + self.min <= 1 + and isinstance(subpattern, WildcardPattern) + and subpattern.min <= 1 + and self.name == subpattern.name + ): + return WildcardPattern( + subpattern.content, + self.min * subpattern.min, + self.max * subpattern.max, + subpattern.name, + ) + return self + + def match(self, node, results=None) -> bool: + """Does this pattern exactly match a node?""" + return self.match_seq([node], results) + + def match_seq(self, nodes, results=None) -> bool: + """Does this pattern exactly match a sequence of nodes?""" + for c, r in self.generate_matches(nodes): + if c == len(nodes): + if results is not None: + results.update(r) + if self.name: + results[self.name] = list(nodes) + return True + return False + + def generate_matches(self, nodes) -> Iterator[tuple[int, _Results]]: + """ + Generator yielding matches for a sequence of nodes. + + Args: + nodes: sequence of nodes + + Yields: + (count, results) tuples where: + count: the match comprises nodes[:count]; + results: dict containing named submatches. + """ + if self.content is None: + # Shortcut for special case (see __init__.__doc__) + for count in range(self.min, 1 + min(len(nodes), self.max)): + r = {} + if self.name: + r[self.name] = nodes[:count] + yield count, r + elif self.name == "bare_name": + yield self._bare_name_matches(nodes) + else: + # The reason for this is that hitting the recursion limit usually + # results in some ugly messages about how RuntimeErrors are being + # ignored. We only have to do this on CPython, though, because other + # implementations don't have this nasty bug in the first place. + if hasattr(sys, "getrefcount"): + save_stderr = sys.stderr + sys.stderr = StringIO() + try: + for count, r in self._recursive_matches(nodes, 0): + if self.name: + r[self.name] = nodes[:count] + yield count, r + except RuntimeError: + # We fall back to the iterative pattern matching scheme if the recursive + # scheme hits the recursion limit. + for count, r in self._iterative_matches(nodes): + if self.name: + r[self.name] = nodes[:count] + yield count, r + finally: + if hasattr(sys, "getrefcount"): + sys.stderr = save_stderr + + def _iterative_matches(self, nodes) -> Iterator[tuple[int, _Results]]: + """Helper to iteratively yield the matches.""" + nodelen = len(nodes) + if 0 >= self.min: + yield 0, {} + + results = [] + # generate matches that use just one alt from self.content + for alt in self.content: + for c, r in generate_matches(alt, nodes): + yield c, r + results.append((c, r)) + + # for each match, iterate down the nodes + while results: + new_results = [] + for c0, r0 in results: + # stop if the entire set of nodes has been matched + if c0 < nodelen and c0 <= self.max: + for alt in self.content: + for c1, r1 in generate_matches(alt, nodes[c0:]): + if c1 > 0: + r = {} + r.update(r0) + r.update(r1) + yield c0 + c1, r + new_results.append((c0 + c1, r)) + results = new_results + + def _bare_name_matches(self, nodes) -> tuple[int, _Results]: + """Special optimized matcher for bare_name.""" + count = 0 + r = {} # type: _Results + done = False + max = len(nodes) + while not done and count < max: + done = True + for leaf in self.content: + if leaf[0].match(nodes[count], r): + count += 1 + done = False + break + assert self.name is not None + r[self.name] = nodes[:count] + return count, r + + def _recursive_matches(self, nodes, count) -> Iterator[tuple[int, _Results]]: + """Helper to recursively yield the matches.""" + assert self.content is not None + if count >= self.min: + yield 0, {} + if count < self.max: + for alt in self.content: + for c0, r0 in generate_matches(alt, nodes): + for c1, r1 in self._recursive_matches(nodes[c0:], count + 1): + r = {} + r.update(r0) + r.update(r1) + yield c0 + c1, r + + +class NegatedPattern(BasePattern): + def __init__(self, content: Optional[BasePattern] = None) -> None: + """ + Initializer. + + The argument is either a pattern or None. If it is None, this + only matches an empty sequence (effectively '$' in regex + lingo). If it is not None, this matches whenever the argument + pattern doesn't have any matches. + """ + if content is not None: + assert isinstance(content, BasePattern), repr(content) + self.content = content + + def match(self, node, results=None) -> bool: + # We never match a node in its entirety + return False + + def match_seq(self, nodes, results=None) -> bool: + # We only match an empty sequence of nodes in its entirety + return len(nodes) == 0 + + def generate_matches(self, nodes: list[NL]) -> Iterator[tuple[int, _Results]]: + if self.content is None: + # Return a match if there is an empty sequence + if len(nodes) == 0: + yield 0, {} + else: + # Return a match if the argument pattern has no matches + for c, r in self.content.generate_matches(nodes): + return + yield 0, {} + + +def generate_matches( + patterns: list[BasePattern], nodes: list[NL] +) -> Iterator[tuple[int, _Results]]: + """ + Generator yielding matches for a sequence of patterns and nodes. + + Args: + patterns: a sequence of patterns + nodes: a sequence of nodes + + Yields: + (count, results) tuples where: + count: the entire sequence of patterns matches nodes[:count]; + results: dict containing named submatches. + """ + if not patterns: + yield 0, {} + else: + p, rest = patterns[0], patterns[1:] + for c0, r0 in p.generate_matches(nodes): + if not rest: + yield c0, r0 + else: + for c1, r1 in generate_matches(rest, nodes[c0:]): + r = {} + r.update(r0) + r.update(r1) + yield c0 + c1, r diff --git a/contrib/python/black/patches/01-fixed-import-tests.patch b/contrib/python/black/patches/01-fixed-import-tests.patch new file mode 100644 index 00000000000..4f1334c8262 --- /dev/null +++ b/contrib/python/black/patches/01-fixed-import-tests.patch @@ -0,0 +1,54 @@ +--- contrib/python/black/blib2to3/pgen2/conv.py (index) ++++ contrib/python/black/blib2to3/pgen2/conv.py (working tree) +@@ -35,1 +35,1 @@ without having to invoke the Python pgen C program. +-from pgen2 import grammar, token ++from ..pgen2 import grammar, token +--- contrib/python/black/blib2to3/pgen2/driver.py (index) ++++ contrib/python/black/blib2to3/pgen2/driver.py (working tree) +@@ -47,6 +47,12 @@ from blib2to3.pgen2.grammar import Grammar + + Path = Union[str, "os.PathLike[str]"] + ++try: ++ import __res ++ IS_ARCADIA = True ++except ImportError: ++ IS_ARCADIA = False ++ + + @dataclass + class ReleaseRange: +@@ -298,7 +304,7 @@ def load_packaged_grammar( + but preserves load_grammar's automatic regeneration behavior when possible. + + """ +- if os.path.isfile(grammar_source): ++ if os.path.isfile(grammar_source) or IS_ARCADIA: + gp = _generate_pickle_name(grammar_source, cache_dir) if cache_dir else None + return load_grammar(grammar_source, gp=gp) + pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir) +--- contrib/python/black/blib2to3/pgen2/pgen.py (index) ++++ contrib/python/black/blib2to3/pgen2/pgen.py (working tree) +@@ -24,6 +24,9 @@ import os + + Path = Union[str, "os.PathLike[str]"] + ++import pkgutil ++import io ++ + + class PgenGrammar(grammar.Grammar): + pass +@@ -39,7 +42,11 @@ class ParserGenerator(object): + def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None: + close_stream = None + if stream is None: +- stream = open(filename, encoding="utf-8") ++ data = pkgutil.get_data("blib2to3", os.path.basename(filename)) ++ if data: ++ stream = io.StringIO(data.decode("utf-8")) ++ else: ++ stream = open(filename, encoding="utf-8") + close_stream = stream.close + self.filename = filename + self.stream = stream diff --git a/contrib/python/black/ya.make b/contrib/python/black/ya.make new file mode 100644 index 00000000000..2a134b4e32a --- /dev/null +++ b/contrib/python/black/ya.make @@ -0,0 +1,86 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(25.1.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/click + contrib/python/mypy-extensions + contrib/python/packaging + contrib/python/pathspec + contrib/python/platformdirs +) + +NO_LINT() + +NO_CHECK_IMPORTS( + black_primer.cli + blackd.* +) + +PY_SRCS( + TOP_LEVEL + _black_version.py + black/__init__.py + black/__main__.py + black/_width_table.py + black/brackets.py + black/cache.py + black/comments.py + black/concurrency.py + black/const.py + black/debug.py + black/files.py + black/handle_ipynb_magics.py + black/linegen.py + black/lines.py + black/mode.py + black/nodes.py + black/numerics.py + black/output.py + black/parsing.py + black/ranges.py + black/report.py + black/resources/__init__.py + black/rusty.py + black/schema.py + black/strings.py + black/trans.py + blackd/__init__.py + blackd/__main__.py + blackd/middlewares.py + blib2to3/__init__.py + blib2to3/pgen2/__init__.py + blib2to3/pgen2/conv.py + blib2to3/pgen2/driver.py + blib2to3/pgen2/grammar.py + blib2to3/pgen2/literals.py + blib2to3/pgen2/parse.py + blib2to3/pgen2/pgen.py + blib2to3/pgen2/token.py + blib2to3/pgen2/tokenize.py + blib2to3/pygram.py + blib2to3/pytree.py +) + +RESOURCE_FILES( + PREFIX contrib/python/black/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt + black/py.typed + black/resources/black.schema.json + blib2to3/Grammar.txt + blib2to3/LICENSE + blib2to3/PatternGrammar.txt + blib2to3/README +) + +END() + +RECURSE( + bin +) diff --git a/contrib/python/marisa-trie/py2/.dist-info/METADATA b/contrib/python/marisa-trie/py2/.dist-info/METADATA new file mode 100644 index 00000000000..b34efb5baf6 --- /dev/null +++ b/contrib/python/marisa-trie/py2/.dist-info/METADATA @@ -0,0 +1,287 @@ +Metadata-Version: 2.1 +Name: marisa-trie +Version: 0.8.0 +Summary: Static memory-efficient and fast Trie-like structures for Python. +Home-page: https://github.com/kmike/marisa-trie +Author: Mikhail Korobov +Author-email: [email protected] +License: MIT +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Scientific/Engineering :: Information Analysis +Classifier: Topic :: Text Processing :: Linguistic +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Description-Content-Type: text/x-rst +License-File: LICENSE +License-File: AUTHORS.rst +Requires-Dist: setuptools +Provides-Extra: test +Requires-Dist: hypothesis ; extra == 'test' +Requires-Dist: pytest ; extra == 'test' +Requires-Dist: readme-renderer ; extra == 'test' + +MARISA Trie +=========== + +.. image:: https://img.shields.io/pypi/pyversions/marisa-trie.svg + :target: https://pypi.python.org/pypi/marisa-trie + +.. image:: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml/badge.svg + :target: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml + +Static memory-efficient Trie-like structures for Python (2.7 and 3.4+) +based on `marisa-trie`_ C++ library. + +String data in a MARISA-trie may take up to 50x-100x less memory than +in a standard Python dict; the raw lookup speed is comparable; trie also +provides fast advanced methods like prefix search. + +.. note:: + + There are official SWIG-based Python bindings included + in C++ library distribution; this package provides alternative + Cython-based pip-installable Python bindings. + +.. _marisa-trie: https://github.com/s-yata/marisa-trie + +Installation +============ + +:: + + python -m pip install -U marisa-trie + +Usage +===== + +See `tutorial`_ and `API`_ for details. + +.. _tutorial: https://marisa-trie.readthedocs.io/en/latest/tutorial.html +.. _API: https://marisa-trie.readthedocs.io/en/latest/api.html + +Current limitations +=================== + +* The library is not tested with mingw32 compiler; +* ``.prefixes()`` method of ``BytesTrie`` and ``RecordTrie`` is quite slow + and doesn't have iterator counterpart; +* ``read()`` and ``write()`` methods don't work with file-like objects + (they work only with real files; pickling works fine for file-like objects); +* there are ``keys()`` and ``items()`` methods but no ``values()`` method. + +License +======= + +Wrapper code is licensed under MIT License. + +Bundled `marisa-trie`_ C++ library is dual-licensed under +LGPL and BSD 2-clause license. + +CHANGES +======= + +0.8.0 (2023-03-25) +------------------ + +* Add ``Trie.iter_prefixes_with_ids()`` method to return ``(prefix, id)`` pairs (#83). +* Rebuild Cython wrapper with Cython 0.29.33 (#88). + +0.7.8 (2022-10-25) +------------------ + +* Added Python 3.11 support. +* Rebuild Cython wrapper with Cython 0.29.32. + +0.7.7 (2021-08-04) +------------------ + +* Restored Python 2.7 support. +* Fixed README image references not working on Windows. + +0.7.6 (2021-07-28) +------------------ + +* Wheels are now published for all platforms. +* Fixed ``ResourceWarning: unclosed file`` in ``setup.py``. +* Run ``black`` on the entire source code. +* Moved the QA/CI to GitHub. +* Rebuild Cython wrapper with Cython 0.29.24. +* Updated ``libmarisa-trie`` to the latest version (0.2.6). +* Fixed failing tests and usage of deprecated methods. +* Expanded supported Python version (2.7, 3.4 - 3.10). + +0.7.5 (2018-04-10) +------------------ + +* Removed redundant ``DeprecationWarning`` messages in ``Trie.save`` and + ``Trie.load``. +* Dropped support for Python 2.6. +* Rebuild Cython wrapper with Cython 0.28.1. + +0.7.4 (2017-03-27) +------------------ + +* Fixed packaging issue, ``MANIFEST.in`` was not updated after ``libmarisa-trie`` + became a submodule. + +0.7.3 (2017-02-14) +------------------ + +* Added ``BinaryTrie`` for storing arbitrary sequences of bytes, e.g. IP + addresses (thanks Tomasz Melcer); +* Deprecated ``Trie.has_keys_with_prefix`` which can be trivially implemented in + terms of ``Trie.iterkeys``; +* Deprecated ``Trie.read`` and ``Trie.write`` which onlywork for "real" files + and duplicate the functionality of ``load`` and ``save``. See issue #31 on + GitHub; +* Updated ``libmarisa-trie`` to the latest version. Yay, 64-bit Windows support. +* Rebuilt Cython wrapper with Cython 0.25.2. + +0.7.2 (2015-04-21) +------------------ + +* packaging issue is fixed. + +0.7.1 (2015-04-21) +------------------ + +* setup.py is switched to setuptools; +* a tiny speedup; +* wrapper is rebuilt with Cython 0.22. + +0.7 (2014-12-15) +---------------- + +* ``trie1 == trie2`` and ``trie1 != trie2`` now work (thanks Sergei Lebedev); +* ``for key in trie:`` is fixed (thanks Sergei Lebedev); +* wrapper is rebuilt with Cython 0.21.1 (thanks Sergei Lebedev); +* https://bitbucket.org/kmike/marisa-trie repo is no longer supported. + +0.6 (2014-02-22) +---------------- + +* New ``Trie`` methods: ``__getitem__``, ``get``, ``items``, ``iteritems``. + ``trie[u'key']`` is now the same as ``trie.key_id(u'key')``. +* small optimization for ``BytesTrie.get``. +* wrapper is rebuilt with Cython 0.20.1. + +0.5.3 (2014-02-08) +------------------ + +* small ``Trie.restore_key`` optimization (it should work 5-15% faster) + +0.5.2 (2014-02-08) +------------------ + +* fix ``Trie.restore_key`` method - it was reading past declared string length; +* rebuild wrapper with Cython 0.20. + +0.5.1 (2013-10-03) +------------------ + +* ``has_keys_with_prefix(prefix)`` method (thanks + `Matt Hickford <https://github.com/matt-hickford>`_) + +0.5 (2013-05-07) +---------------- + +* ``BytesTrie.iterkeys``, ``BytesTrie.iteritems``, + ``RecordTrie.iterkeys`` and ``RecordTrie.iteritems`` methods; +* wrapper is rebuilt with Cython 0.19; +* ``value_separator`` parameter for ``BytesTrie`` and ``RecordTrie``. + +0.4 (2013-02-28) +---------------- + +* improved trie building: ``weights`` optional parameter; +* improved trie building: unnecessary input sorting is removed; +* wrapper is rebuilt with Cython 0.18; +* bundled marisa-trie C++ library is updated to svn r133. + +0.3.8 (2013-01-03) +------------------ + +* Rebuild wrapper with Cython pre-0.18; +* update benchmarks. + +0.3.7 (2012-09-21) +------------------ + +* Update bundled marisa-trie C++ library (this may fix more mingw issues); +* Python 3.3 support is back. + +0.3.6 (2012-09-05) +------------------ + +* much faster (3x-7x) ``.items()`` and ``.keys()`` methods for all tries; + faster (up to 3x) ``.prefixes()`` method for ``Trie``. + +0.3.5 (2012-08-30) +------------------ + +* Pickling of RecordTrie is fixed (thanks lazarou for the report); +* error messages should become more useful. + +0.3.4 (2012-08-29) +------------------ + +* Issues with mingw32 should be resolved (thanks Susumu Yata). + +0.3.3 (2012-08-27) +------------------ + +* ``.get(key, default=None)`` method for ``BytesTrie`` and ``RecordTrie``; +* small README improvements. + +0.3.2 (2012-08-26) +------------------ + +* Small code cleanup; +* ``load``, ``read`` and ``mmap`` methods returns 'self'; +* I can't run tests (via tox) under Python 3.3 so it is + removed from supported versions for now. + +0.3.1 (2012-08-23) +------------------ + +* ``.prefixes()`` support for RecordTrie and BytesTrie. + +0.3 (2012-08-23) +---------------- + +* RecordTrie and BytesTrie are introduced; +* IntTrie class is removed (probably temporary?); +* dumps/loads methods are renamed to tobytes/frombytes; +* benchmark & tests improvements; +* support for MARISA-trie config options is added. + +0.2 (2012-08-19) +------------------ + +* Pickling/unpickling support; +* dumps/loads methods; +* python 3.3 workaround; +* improved tests; +* benchmarks. + +0.1 (2012-08-17) +---------------- + +Initial release. diff --git a/contrib/python/marisa-trie/py2/.dist-info/top_level.txt b/contrib/python/marisa-trie/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..62049767090 --- /dev/null +++ b/contrib/python/marisa-trie/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +marisa_trie diff --git a/contrib/python/marisa-trie/py2/.yandex_meta/yamaker.yaml b/contrib/python/marisa-trie/py2/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..216bd445a8d --- /dev/null +++ b/contrib/python/marisa-trie/py2/.yandex_meta/yamaker.yaml @@ -0,0 +1,3 @@ +copy: + - marisa-trie/* + - src/* diff --git a/contrib/python/marisa-trie/py2/AUTHORS.rst b/contrib/python/marisa-trie/py2/AUTHORS.rst new file mode 100644 index 00000000000..9df75381ed6 --- /dev/null +++ b/contrib/python/marisa-trie/py2/AUTHORS.rst @@ -0,0 +1,7 @@ +Authors and contributors, in no particular order: + +* Mikhail Korobov <[email protected]> +* `Matt Hickford <https://github.com/matt-hickford>`_ +* Sergei Lebedev <[email protected]> +* Tomasz Melcer <https://github.com/liori> +* `Mickaël Schoentgen <https://github.com/BoboTiG>` diff --git a/contrib/python/marisa-trie/py2/LICENSE b/contrib/python/marisa-trie/py2/LICENSE new file mode 100644 index 00000000000..9cf5bc8acd7 --- /dev/null +++ b/contrib/python/marisa-trie/py2/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) marisa-trie authors and contributors, 2012-2023 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR +A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/marisa-trie/py2/README.rst b/contrib/python/marisa-trie/py2/README.rst new file mode 100644 index 00000000000..56485ebe205 --- /dev/null +++ b/contrib/python/marisa-trie/py2/README.rst @@ -0,0 +1,56 @@ +MARISA Trie +=========== + +.. image:: https://img.shields.io/pypi/pyversions/marisa-trie.svg + :target: https://pypi.python.org/pypi/marisa-trie + +.. image:: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml/badge.svg + :target: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml + +Static memory-efficient Trie-like structures for Python (2.7 and 3.4+) +based on `marisa-trie`_ C++ library. + +String data in a MARISA-trie may take up to 50x-100x less memory than +in a standard Python dict; the raw lookup speed is comparable; trie also +provides fast advanced methods like prefix search. + +.. note:: + + There are official SWIG-based Python bindings included + in C++ library distribution; this package provides alternative + Cython-based pip-installable Python bindings. + +.. _marisa-trie: https://github.com/s-yata/marisa-trie + +Installation +============ + +:: + + python -m pip install -U marisa-trie + +Usage +===== + +See `tutorial`_ and `API`_ for details. + +.. _tutorial: https://marisa-trie.readthedocs.io/en/latest/tutorial.html +.. _API: https://marisa-trie.readthedocs.io/en/latest/api.html + +Current limitations +=================== + +* The library is not tested with mingw32 compiler; +* ``.prefixes()`` method of ``BytesTrie`` and ``RecordTrie`` is quite slow + and doesn't have iterator counterpart; +* ``read()`` and ``write()`` methods don't work with file-like objects + (they work only with real files; pickling works fine for file-like objects); +* there are ``keys()`` and ``items()`` methods but no ``values()`` method. + +License +======= + +Wrapper code is licensed under MIT License. + +Bundled `marisa-trie`_ C++ library is dual-licensed under +LGPL and BSD 2-clause license. diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/agent.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/agent.h new file mode 100644 index 00000000000..c72dea755d4 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/agent.h @@ -0,0 +1,73 @@ +#ifndef MARISA_AGENT_H_ +#define MARISA_AGENT_H_ + +#include "marisa/key.h" +#include "marisa/query.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class State; + +} // namespace trie +} // namespace grimoire + +class Agent { + public: + Agent(); + ~Agent(); + + const Query &query() const { + return query_; + } + const Key &key() const { + return key_; + } + + void set_query(const char *str); + void set_query(const char *ptr, std::size_t length); + void set_query(std::size_t key_id); + + const grimoire::trie::State &state() const { + return *state_; + } + grimoire::trie::State &state() { + return *state_; + } + + void set_key(const char *str) { + MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR); + key_.set_str(str); + } + void set_key(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_.set_str(ptr, length); + } + void set_key(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_.set_id(id); + } + + bool has_state() const { + return state_.get() != NULL; + } + void init_state(); + + void clear(); + void swap(Agent &rhs); + + private: + Query query_; + Key key_; + scoped_ptr<grimoire::trie::State> state_; + + // Disallows copy and assignment. + Agent(const Agent &); + Agent &operator=(const Agent &); +}; + +} // namespace marisa + +#endif // MARISA_AGENT_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/base.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/base.h new file mode 100644 index 00000000000..ffcdc5bd8cc --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/base.h @@ -0,0 +1,196 @@ +#ifndef MARISA_BASE_H_ +#define MARISA_BASE_H_ + +// Old Visual C++ does not provide stdint.h. +#ifndef _MSC_VER + #include <stdint.h> +#endif // _MSC_VER + +#ifdef __cplusplus + #include <cstddef> +#else // __cplusplus + #include <stddef.h> +#endif // __cplusplus + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#ifdef _MSC_VER +typedef unsigned __int8 marisa_uint8; +typedef unsigned __int16 marisa_uint16; +typedef unsigned __int32 marisa_uint32; +typedef unsigned __int64 marisa_uint64; +#else // _MSC_VER +typedef uint8_t marisa_uint8; +typedef uint16_t marisa_uint16; +typedef uint32_t marisa_uint32; +typedef uint64_t marisa_uint64; +#endif // _MSC_VER + +#if defined(_WIN64) || defined(__amd64__) || defined(__x86_64__) || \ + defined(__ia64__) || defined(__ppc64__) || defined(__powerpc64__) || \ + defined(__sparc64__) || defined(__mips64__) || defined(__aarch64__) || \ + defined(__s390x__) + #define MARISA_WORD_SIZE 64 +#else // defined(_WIN64), etc. + #define MARISA_WORD_SIZE 32 +#endif // defined(_WIN64), etc. + +//#define MARISA_WORD_SIZE (sizeof(void *) * 8) + +#define MARISA_UINT8_MAX ((marisa_uint8)~(marisa_uint8)0) +#define MARISA_UINT16_MAX ((marisa_uint16)~(marisa_uint16)0) +#define MARISA_UINT32_MAX ((marisa_uint32)~(marisa_uint32)0) +#define MARISA_UINT64_MAX ((marisa_uint64)~(marisa_uint64)0) +#define MARISA_SIZE_MAX ((size_t)~(size_t)0) + +#define MARISA_INVALID_LINK_ID MARISA_UINT32_MAX +#define MARISA_INVALID_KEY_ID MARISA_UINT32_MAX +#define MARISA_INVALID_EXTRA (MARISA_UINT32_MAX >> 8) + +// Error codes are defined as members of marisa_error_code. This library throws +// an exception with one of the error codes when an error occurs. +typedef enum marisa_error_code_ { + // MARISA_OK means that a requested operation has succeeded. In practice, an + // exception never has MARISA_OK because it is not an error. + MARISA_OK = 0, + + // MARISA_STATE_ERROR means that an object was not ready for a requested + // operation. For example, an operation to modify a fixed vector throws an + // exception with MARISA_STATE_ERROR. + MARISA_STATE_ERROR = 1, + + // MARISA_NULL_ERROR means that an invalid NULL pointer has been given. + MARISA_NULL_ERROR = 2, + + // MARISA_BOUND_ERROR means that an operation has tried to access an out of + // range address. + MARISA_BOUND_ERROR = 3, + + // MARISA_RANGE_ERROR means that an out of range value has appeared in + // operation. + MARISA_RANGE_ERROR = 4, + + // MARISA_CODE_ERROR means that an undefined code has appeared in operation. + MARISA_CODE_ERROR = 5, + + // MARISA_RESET_ERROR means that a smart pointer has tried to reset itself. + MARISA_RESET_ERROR = 6, + + // MARISA_SIZE_ERROR means that a size has exceeded a library limitation. + MARISA_SIZE_ERROR = 7, + + // MARISA_MEMORY_ERROR means that a memory allocation has failed. + MARISA_MEMORY_ERROR = 8, + + // MARISA_IO_ERROR means that an I/O operation has failed. + MARISA_IO_ERROR = 9, + + // MARISA_FORMAT_ERROR means that input was in invalid format. + MARISA_FORMAT_ERROR = 10, +} marisa_error_code; + +// Min/max values, flags and masks for dictionary settings are defined below. +// Please note that unspecified settings will be replaced with the default +// settings. For example, 0 is equivalent to (MARISA_DEFAULT_NUM_TRIES | +// MARISA_DEFAULT_TRIE | MARISA_DEFAULT_TAIL | MARISA_DEFAULT_ORDER). + +// A dictionary consists of 3 tries in default. Usually more tries make a +// dictionary space-efficient but time-inefficient. +typedef enum marisa_num_tries_ { + MARISA_MIN_NUM_TRIES = 0x00001, + MARISA_MAX_NUM_TRIES = 0x0007F, + MARISA_DEFAULT_NUM_TRIES = 0x00003, +} marisa_num_tries; + +// This library uses a cache technique to accelerate search functions. The +// following enumerated type marisa_cache_level gives a list of available cache +// size options. A larger cache enables faster search but takes a more space. +typedef enum marisa_cache_level_ { + MARISA_HUGE_CACHE = 0x00080, + MARISA_LARGE_CACHE = 0x00100, + MARISA_NORMAL_CACHE = 0x00200, + MARISA_SMALL_CACHE = 0x00400, + MARISA_TINY_CACHE = 0x00800, + MARISA_DEFAULT_CACHE = MARISA_NORMAL_CACHE +} marisa_cache_level; + +// This library provides 2 kinds of TAIL implementations. +typedef enum marisa_tail_mode_ { + // MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is + // available if and only if the last labels do not contain a NULL character. + // If MARISA_TEXT_TAIL is specified and a NULL character exists in the last + // labels, the setting is automatically switched to MARISA_BINARY_TAIL. + MARISA_TEXT_TAIL = 0x01000, + + // MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses + // a bit vector to detect the end of a sequence, instead of NULL characters. + // So, MARISA_BINARY_TAIL requires a larger space if the average length of + // labels is greater than 8. + MARISA_BINARY_TAIL = 0x02000, + + MARISA_DEFAULT_TAIL = MARISA_TEXT_TAIL, +} marisa_tail_mode; + +// The arrangement of nodes affects the time cost of matching and the order of +// predictive search. +typedef enum marisa_node_order_ { + // MARISA_LABEL_ORDER arranges nodes in ascending label order. + // MARISA_LABEL_ORDER is useful if an application needs to predict keys in + // label order. + MARISA_LABEL_ORDER = 0x10000, + + // MARISA_WEIGHT_ORDER arranges nodes in descending weight order. + // MARISA_WEIGHT_ORDER is generally a better choice because it enables faster + // matching. + MARISA_WEIGHT_ORDER = 0x20000, + + MARISA_DEFAULT_ORDER = MARISA_WEIGHT_ORDER, +} marisa_node_order; + +typedef enum marisa_config_mask_ { + MARISA_NUM_TRIES_MASK = 0x0007F, + MARISA_CACHE_LEVEL_MASK = 0x00F80, + MARISA_TAIL_MODE_MASK = 0x0F000, + MARISA_NODE_ORDER_MASK = 0xF0000, + MARISA_CONFIG_MASK = 0xFFFFF +} marisa_config_mask; + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#ifdef __cplusplus + +// `std::swap` is in <utility> since C++ 11 but in <algorithm> in C++ 98: +#if __cplusplus >= 201103L + #include <utility> +#else + #include <algorithm> +#endif +namespace marisa { + +typedef ::marisa_uint8 UInt8; +typedef ::marisa_uint16 UInt16; +typedef ::marisa_uint32 UInt32; +typedef ::marisa_uint64 UInt64; + +typedef ::marisa_error_code ErrorCode; + +typedef ::marisa_cache_level CacheLevel; +typedef ::marisa_tail_mode TailMode; +typedef ::marisa_node_order NodeOrder; + +using std::swap; + +} // namespace marisa +#endif // __cplusplus + +#ifdef __cplusplus + #include "marisa/exception.h" + #include "marisa/scoped-ptr.h" + #include "marisa/scoped-array.h" +#endif // __cplusplus + +#endif // MARISA_BASE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/exception.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/exception.h new file mode 100644 index 00000000000..c4a93aa4978 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/exception.h @@ -0,0 +1,82 @@ +#ifndef MARISA_EXCEPTION_H_ +#define MARISA_EXCEPTION_H_ + +#include <exception> + +#include "marisa/base.h" + +namespace marisa { + +// An exception object keeps a filename, a line number, an error code and an +// error message. The message format is as follows: +// "__FILE__:__LINE__: error_code: error_message" +class Exception : public std::exception { + public: + Exception(const char *filename, int line, + ErrorCode error_code, const char *error_message) + : std::exception(), filename_(filename), line_(line), + error_code_(error_code), error_message_(error_message) {} + Exception(const Exception &ex) + : std::exception(), filename_(ex.filename_), line_(ex.line_), + error_code_(ex.error_code_), error_message_(ex.error_message_) {} + virtual ~Exception() {} + + Exception &operator=(const Exception &rhs) { + filename_ = rhs.filename_; + line_ = rhs.line_; + error_code_ = rhs.error_code_; + error_message_ = rhs.error_message_; + return *this; + } + + const char *filename() const { + return filename_; + } + int line() const { + return line_; + } + ErrorCode error_code() const { + return error_code_; + } + const char *error_message() const { + return error_message_; + } + + virtual const char *what() const noexcept { + return error_message_; + } + + private: + const char *filename_; + int line_; + ErrorCode error_code_; + const char *error_message_; +}; + +// These macros are used to convert a line number to a string constant. +#define MARISA_INT_TO_STR(value) #value +#define MARISA_LINE_TO_STR(line) MARISA_INT_TO_STR(line) +#define MARISA_LINE_STR MARISA_LINE_TO_STR(__LINE__) + +// MARISA_THROW throws an exception with a filename, a line number, an error +// code and an error message. The message format is as follows: +// "__FILE__:__LINE__: error_code: error_message" +#define MARISA_THROW(error_code, error_message) \ + (throw marisa::Exception(__FILE__, __LINE__, error_code, \ + __FILE__ ":" MARISA_LINE_STR ": " #error_code ": " error_message)) + +// MARISA_THROW_IF throws an exception if `condition' is true. +#define MARISA_THROW_IF(condition, error_code) \ + (void)((!(condition)) || (MARISA_THROW(error_code, #condition), 0)) + +// MARISA_DEBUG_IF is ignored if _DEBUG is undefined. So, it is useful for +// debugging time-critical codes. +#ifdef _DEBUG + #define MARISA_DEBUG_IF(cond, error_code) MARISA_THROW_IF(cond, error_code) +#else + #define MARISA_DEBUG_IF(cond, error_code) +#endif + +} // namespace marisa + +#endif // MARISA_EXCEPTION_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/iostream.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/iostream.h new file mode 100644 index 00000000000..05d139a23b8 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/iostream.h @@ -0,0 +1,18 @@ +#ifndef MARISA_IOSTREAM_H_ +#define MARISA_IOSTREAM_H_ + +#include <iosfwd> + +namespace marisa { + +class Trie; + +std::istream &read(std::istream &stream, Trie *trie); +std::ostream &write(std::ostream &stream, const Trie &trie); + +std::istream &operator>>(std::istream &stream, Trie &trie); +std::ostream &operator<<(std::ostream &stream, const Trie &trie); + +} // namespace marisa + +#endif // MARISA_IOSTREAM_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/key.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/key.h new file mode 100644 index 00000000000..8b2cf8fcf8a --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/key.h @@ -0,0 +1,85 @@ +#ifndef MARISA_KEY_H_ +#define MARISA_KEY_H_ + +#include "marisa/base.h" + +namespace marisa { + +class Key { + public: + Key() : ptr_(NULL), length_(0), union_() { + union_.id = 0; + } + Key(const Key &key) + : ptr_(key.ptr_), length_(key.length_), union_(key.union_) {} + + Key &operator=(const Key &key) { + ptr_ = key.ptr_; + length_ = key.length_; + union_ = key.union_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return ptr_[i]; + } + + void set_str(const char *str) { + MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR); + std::size_t length = 0; + while (str[length] != '\0') { + ++length; + } + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = str; + length_ = (UInt32)length; + } + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr; + length_ = (UInt32)length; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + union_.id = (UInt32)id; + } + void set_weight(float weight) { + union_.weight = weight; + } + + const char *ptr() const { + return ptr_; + } + std::size_t length() const { + return length_; + } + std::size_t id() const { + return union_.id; + } + float weight() const { + return union_.weight; + } + + void clear() { + Key().swap(*this); + } + void swap(Key &rhs) { + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(length_, rhs.length_); + marisa::swap(union_.id, rhs.union_.id); + } + + private: + const char *ptr_; + UInt32 length_; + union Union { + UInt32 id; + float weight; + } union_; +}; + +} // namespace marisa + +#endif // MARISA_KEY_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/keyset.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/keyset.h new file mode 100644 index 00000000000..e575015c0c3 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/keyset.h @@ -0,0 +1,80 @@ +#ifndef MARISA_KEYSET_H_ +#define MARISA_KEYSET_H_ + +#include "marisa/key.h" + +namespace marisa { + +class Keyset { + public: + enum { + BASE_BLOCK_SIZE = 4096, + EXTRA_BLOCK_SIZE = 1024, + KEY_BLOCK_SIZE = 256 + }; + + Keyset(); + + void push_back(const Key &key); + void push_back(const Key &key, char end_marker); + + void push_back(const char *str); + void push_back(const char *ptr, std::size_t length, float weight = 1.0); + + const Key &operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return key_blocks_[i / KEY_BLOCK_SIZE][i % KEY_BLOCK_SIZE]; + } + Key &operator[](std::size_t i) { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return key_blocks_[i / KEY_BLOCK_SIZE][i % KEY_BLOCK_SIZE]; + } + + std::size_t num_keys() const { + return size_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + std::size_t total_length() const { + return total_length_; + } + + void reset(); + + void clear(); + void swap(Keyset &rhs); + + private: + scoped_array<scoped_array<char> > base_blocks_; + std::size_t base_blocks_size_; + std::size_t base_blocks_capacity_; + scoped_array<scoped_array<char> > extra_blocks_; + std::size_t extra_blocks_size_; + std::size_t extra_blocks_capacity_; + scoped_array<scoped_array<Key> > key_blocks_; + std::size_t key_blocks_size_; + std::size_t key_blocks_capacity_; + char *ptr_; + std::size_t avail_; + std::size_t size_; + std::size_t total_length_; + + char *reserve(std::size_t size); + + void append_base_block(); + void append_extra_block(std::size_t size); + void append_key_block(); + + // Disallows copy and assignment. + Keyset(const Keyset &); + Keyset &operator=(const Keyset &); +}; + +} // namespace marisa + +#endif // MARISA_KEYSET_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/query.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/query.h new file mode 100644 index 00000000000..316e741a8fa --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/query.h @@ -0,0 +1,71 @@ +#ifndef MARISA_QUERY_H_ +#define MARISA_QUERY_H_ + +#include "marisa/base.h" + +namespace marisa { + +class Query { + public: + Query() : ptr_(NULL), length_(0), id_(0) {} + Query(const Query &query) + : ptr_(query.ptr_), length_(query.length_), id_(query.id_) {} + + Query &operator=(const Query &query) { + ptr_ = query.ptr_; + length_ = query.length_; + id_ = query.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return ptr_[i]; + } + + void set_str(const char *str) { + MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR); + std::size_t length = 0; + while (str[length] != '\0') { + ++length; + } + ptr_ = str; + length_ = length; + } + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + ptr_ = ptr; + length_ = length; + } + void set_id(std::size_t id) { + id_ = id; + } + + const char *ptr() const { + return ptr_; + } + std::size_t length() const { + return length_; + } + std::size_t id() const { + return id_; + } + + void clear() { + Query().swap(*this); + } + void swap(Query &rhs) { + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(length_, rhs.length_); + marisa::swap(id_, rhs.id_); + } + + private: + const char *ptr_; + std::size_t length_; + std::size_t id_; +}; + +} // namespace marisa + +#endif // MARISA_QUERY_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/scoped-array.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/scoped-array.h new file mode 100644 index 00000000000..12b5b9e7f83 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/scoped-array.h @@ -0,0 +1,48 @@ +#ifndef MARISA_SCOPED_ARRAY_H_ +#define MARISA_SCOPED_ARRAY_H_ + +#include "marisa/base.h" + +namespace marisa { + +template <typename T> +class scoped_array { + public: + scoped_array() : array_(NULL) {} + explicit scoped_array(T *array) : array_(array) {} + + ~scoped_array() { + delete [] array_; + } + + void reset(T *array = NULL) { + MARISA_THROW_IF((array != NULL) && (array == array_), MARISA_RESET_ERROR); + scoped_array(array).swap(*this); + } + + T &operator[](std::size_t i) const { + MARISA_DEBUG_IF(array_ == NULL, MARISA_STATE_ERROR); + return array_[i]; + } + T *get() const { + return array_; + } + + void clear() { + scoped_array().swap(*this); + } + void swap(scoped_array &rhs) { + marisa::swap(array_, rhs.array_); + } + + private: + T *array_; + + // Disallows copy and assignment. + scoped_array(const scoped_array &); + scoped_array &operator=(const scoped_array &); +}; + +} // namespace marisa + +#endif // MARISA_SCOPED_ARRAY_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/scoped-ptr.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/scoped-ptr.h new file mode 100644 index 00000000000..63d7a3dcc92 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/scoped-ptr.h @@ -0,0 +1,52 @@ +#ifndef MARISA_SCOPED_PTR_H_ +#define MARISA_SCOPED_PTR_H_ + +#include "marisa/base.h" + +namespace marisa { + +template <typename T> +class scoped_ptr { + public: + scoped_ptr() : ptr_(NULL) {} + explicit scoped_ptr(T *ptr) : ptr_(ptr) {} + + ~scoped_ptr() { + delete ptr_; + } + + void reset(T *ptr = NULL) { + MARISA_THROW_IF((ptr != NULL) && (ptr == ptr_), MARISA_RESET_ERROR); + scoped_ptr(ptr).swap(*this); + } + + T &operator*() const { + MARISA_DEBUG_IF(ptr_ == NULL, MARISA_STATE_ERROR); + return *ptr_; + } + T *operator->() const { + MARISA_DEBUG_IF(ptr_ == NULL, MARISA_STATE_ERROR); + return ptr_; + } + T *get() const { + return ptr_; + } + + void clear() { + scoped_ptr().swap(*this); + } + void swap(scoped_ptr &rhs) { + marisa::swap(ptr_, rhs.ptr_); + } + + private: + T *ptr_; + + // Disallows copy and assignment. + scoped_ptr(const scoped_ptr &); + scoped_ptr &operator=(const scoped_ptr &); +}; + +} // namespace marisa + +#endif // MARISA_SCOPED_PTR_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/stdio.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/stdio.h new file mode 100644 index 00000000000..ee9a7b7795f --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/stdio.h @@ -0,0 +1,15 @@ +#ifndef MARISA_MYSTDIO_H_ +#define MARISA_MYSTDIO_H_ + +#include <cstdio> + +namespace marisa { + +class Trie; + +void fread(std::FILE *file, Trie *trie); +void fwrite(std::FILE *file, const Trie &trie); + +} // namespace marisa + +#endif // MARISA_MYSTDIO_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/trie.h b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/trie.h new file mode 100644 index 00000000000..30f3c686b26 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/include/marisa/trie.h @@ -0,0 +1,64 @@ +#ifndef MARISA_TRIE_H_ +#define MARISA_TRIE_H_ + +#include "marisa/keyset.h" +#include "marisa/agent.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class LoudsTrie; + +} // namespace trie +} // namespace grimoire + +class Trie { + friend class TrieIO; + + public: + Trie(); + ~Trie(); + + void build(Keyset &keyset, int config_flags = 0); + + void mmap(const char *filename); + void map(const void *ptr, std::size_t size); + + void load(const char *filename); + void read(int fd); + + void save(const char *filename) const; + void write(int fd) const; + + bool lookup(Agent &agent) const; + void reverse_lookup(Agent &agent) const; + bool common_prefix_search(Agent &agent) const; + bool predictive_search(Agent &agent) const; + + std::size_t num_tries() const; + std::size_t num_keys() const; + std::size_t num_nodes() const; + + TailMode tail_mode() const; + NodeOrder node_order() const; + + bool empty() const; + std::size_t size() const; + std::size_t total_size() const; + std::size_t io_size() const; + + void clear(); + void swap(Trie &rhs); + + private: + scoped_ptr<grimoire::trie::LoudsTrie> trie_; + + // Disallows copy and assignment. + Trie(const Trie &); + Trie &operator=(const Trie &); +}; + +} // namespace marisa + +#endif // MARISA_TRIE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/agent.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/agent.cc new file mode 100644 index 00000000000..7fa7cb1df91 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/agent.cc @@ -0,0 +1,51 @@ +#include <new> + +#include "marisa/agent.h" +#include "marisa/grimoire/trie.h" + +namespace marisa { + +Agent::Agent() : query_(), key_(), state_() {} + +Agent::~Agent() {} + +void Agent::set_query(const char *str) { + MARISA_THROW_IF(str == NULL, MARISA_NULL_ERROR); + if (state_.get() != NULL) { + state_->reset(); + } + query_.set_str(str); +} + +void Agent::set_query(const char *ptr, std::size_t length) { + MARISA_THROW_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + if (state_.get() != NULL) { + state_->reset(); + } + query_.set_str(ptr, length); +} + +void Agent::set_query(std::size_t key_id) { + if (state_.get() != NULL) { + state_->reset(); + } + query_.set_id(key_id); +} + +void Agent::init_state() { + MARISA_THROW_IF(state_.get() != NULL, MARISA_STATE_ERROR); + state_.reset(new (std::nothrow) grimoire::State); + MARISA_THROW_IF(state_.get() == NULL, MARISA_MEMORY_ERROR); +} + +void Agent::clear() { + Agent().swap(*this); +} + +void Agent::swap(Agent &rhs) { + query_.swap(rhs.query_); + key_.swap(rhs.key_); + state_.swap(rhs.state_); +} + +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/algorithm.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/algorithm.h new file mode 100644 index 00000000000..970c09f0eaf --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/algorithm.h @@ -0,0 +1,26 @@ +#ifndef MARISA_GRIMOIRE_ALGORITHM_H_ +#define MARISA_GRIMOIRE_ALGORITHM_H_ + +#include "marisa/grimoire/algorithm/sort.h" + +namespace marisa { +namespace grimoire { + +class Algorithm { + public: + Algorithm() {} + + template <typename Iterator> + std::size_t sort(Iterator begin, Iterator end) const { + return algorithm::sort(begin, end); + } + + private: + Algorithm(const Algorithm &); + Algorithm &operator=(const Algorithm &); +}; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_ALGORITHM_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/algorithm/sort.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/algorithm/sort.h new file mode 100644 index 00000000000..5f3b6526bda --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/algorithm/sort.h @@ -0,0 +1,196 @@ +#ifndef MARISA_GRIMOIRE_ALGORITHM_SORT_H_ +#define MARISA_GRIMOIRE_ALGORITHM_SORT_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace algorithm { +namespace details { + +enum { + MARISA_INSERTION_SORT_THRESHOLD = 10 +}; + +template <typename T> +int get_label(const T &unit, std::size_t depth) { + MARISA_DEBUG_IF(depth > unit.length(), MARISA_BOUND_ERROR); + + return (depth < unit.length()) ? (int)(UInt8)unit[depth] : -1; +} + +template <typename T> +int median(const T &a, const T &b, const T &c, std::size_t depth) { + const int x = get_label(a, depth); + const int y = get_label(b, depth); + const int z = get_label(c, depth); + if (x < y) { + if (y < z) { + return y; + } else if (x < z) { + return z; + } + return x; + } else if (x < z) { + return x; + } else if (y < z) { + return z; + } + return y; +} + +template <typename T> +int compare(const T &lhs, const T &rhs, std::size_t depth) { + for (std::size_t i = depth; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return 1; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] - (UInt8)rhs[i]; + } + } + if (lhs.length() == rhs.length()) { + return 0; + } + return (lhs.length() < rhs.length()) ? -1 : 1; +} + +template <typename Iterator> +std::size_t insertion_sort(Iterator l, Iterator r, std::size_t depth) { + MARISA_DEBUG_IF(l > r, MARISA_BOUND_ERROR); + + std::size_t count = 1; + for (Iterator i = l + 1; i < r; ++i) { + int result = 0; + for (Iterator j = i; j > l; --j) { + result = compare(*(j - 1), *j, depth); + if (result <= 0) { + break; + } + marisa::swap(*(j - 1), *j); + } + if (result != 0) { + ++count; + } + } + return count; +} + +template <typename Iterator> +std::size_t sort(Iterator l, Iterator r, std::size_t depth) { + MARISA_DEBUG_IF(l > r, MARISA_BOUND_ERROR); + + std::size_t count = 0; + while ((r - l) > MARISA_INSERTION_SORT_THRESHOLD) { + Iterator pl = l; + Iterator pr = r; + Iterator pivot_l = l; + Iterator pivot_r = r; + + const int pivot = median(*l, *(l + (r - l) / 2), *(r - 1), depth); + for ( ; ; ) { + while (pl < pr) { + const int label = get_label(*pl, depth); + if (label > pivot) { + break; + } else if (label == pivot) { + marisa::swap(*pl, *pivot_l); + ++pivot_l; + } + ++pl; + } + while (pl < pr) { + const int label = get_label(*--pr, depth); + if (label < pivot) { + break; + } else if (label == pivot) { + marisa::swap(*pr, *--pivot_r); + } + } + if (pl >= pr) { + break; + } + marisa::swap(*pl, *pr); + ++pl; + } + while (pivot_l > l) { + marisa::swap(*--pivot_l, *--pl); + } + while (pivot_r < r) { + marisa::swap(*pivot_r, *pr); + ++pivot_r; + ++pr; + } + + if (((pl - l) > (pr - pl)) || ((r - pr) > (pr - pl))) { + if ((pr - pl) == 1) { + ++count; + } else if ((pr - pl) > 1) { + if (pivot == -1) { + ++count; + } else { + count += sort(pl, pr, depth + 1); + } + } + + if ((pl - l) < (r - pr)) { + if ((pl - l) == 1) { + ++count; + } else if ((pl - l) > 1) { + count += sort(l, pl, depth); + } + l = pr; + } else { + if ((r - pr) == 1) { + ++count; + } else if ((r - pr) > 1) { + count += sort(pr, r, depth); + } + r = pl; + } + } else { + if ((pl - l) == 1) { + ++count; + } else if ((pl - l) > 1) { + count += sort(l, pl, depth); + } + + if ((r - pr) == 1) { + ++count; + } else if ((r - pr) > 1) { + count += sort(pr, r, depth); + } + + l = pl, r = pr; + if ((pr - pl) == 1) { + ++count; + } else if ((pr - pl) > 1) { + if (pivot == -1) { + l = r; + ++count; + } else { + ++depth; + } + } + } + } + + if ((r - l) > 1) { + count += insertion_sort(l, r, depth); + } + return count; +} + +} // namespace details + +template <typename Iterator> +std::size_t sort(Iterator begin, Iterator end) { + MARISA_DEBUG_IF(begin > end, MARISA_BOUND_ERROR); + return details::sort(begin, end, 0); +}; + +} // namespace algorithm +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_ALGORITHM_SORT_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/intrin.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/intrin.h new file mode 100644 index 00000000000..77b4e9980f5 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/intrin.h @@ -0,0 +1,138 @@ +#ifndef MARISA_GRIMOIRE_INTRIN_H_ +#define MARISA_GRIMOIRE_INTRIN_H_ + +#include "marisa/base.h" + +#if defined(__x86_64__) || defined(_M_X64) + #define MARISA_X64 +#elif defined(__i386__) || defined(_M_IX86) + #define MARISA_X86 +#else // defined(__i386__) || defined(_M_IX86) + #ifdef MARISA_USE_BMI2 + #undef MARISA_USE_BMI2 + #endif // MARISA_USE_BMI2 + #ifdef MARISA_USE_BMI + #undef MARISA_USE_BMI + #endif // MARISA_USE_BMI + #ifdef MARISA_USE_POPCNT + #undef MARISA_USE_POPCNT + #endif // MARISA_USE_POPCNT + #ifdef MARISA_USE_SSE4A + #undef MARISA_USE_SSE4A + #endif // MARISA_USE_SSE4A + #ifdef MARISA_USE_SSE4 + #undef MARISA_USE_SSE4 + #endif // MARISA_USE_SSE4 + #ifdef MARISA_USE_SSE4_2 + #undef MARISA_USE_SSE4_2 + #endif // MARISA_USE_SSE4_2 + #ifdef MARISA_USE_SSE4_1 + #undef MARISA_USE_SSE4_1 + #endif // MARISA_USE_SSE4_1 + #ifdef MARISA_USE_SSSE3 + #undef MARISA_USE_SSSE3 + #endif // MARISA_USE_SSSE3 + #ifdef MARISA_USE_SSE3 + #undef MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifdef MARISA_USE_SSE2 + #undef MARISA_USE_SSE2 + #endif // MARISA_USE_SSE2 +#endif // defined(__i386__) || defined(_M_IX86) + +#ifdef MARISA_USE_BMI2 + #ifndef MARISA_USE_BMI + #define MARISA_USE_BMI + #endif // MARISA_USE_BMI + #ifdef _MSC_VER + #include <immintrin.h> + #else // _MSC_VER + #include <x86intrin.h> + #endif // _MSC_VER +#endif // MARISA_USE_BMI2 + +#ifdef MARISA_USE_BMI + #ifndef MARISA_USE_SSE4 + #define MARISA_USE_SSE4 + #endif // MARISA_USE_SSE4 +#endif // MARISA_USE_BMI + +#ifdef MARISA_USE_SSE4A + #ifndef MARISA_USE_SSE3 + #define MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifndef MARISA_USE_POPCNT + #define MARISA_USE_POPCNT + #endif // MARISA_USE_POPCNT +#endif // MARISA_USE_SSE4A + +#ifdef MARISA_USE_SSE4 + #ifndef MARISA_USE_SSE4_2 + #define MARISA_USE_SSE4_2 + #endif // MARISA_USE_SSE4_2 +#endif // MARISA_USE_SSE4 + +#ifdef MARISA_USE_SSE4_2 + #ifndef MARISA_USE_SSE4_1 + #define MARISA_USE_SSE4_1 + #endif // MARISA_USE_SSE4_1 + #ifndef MARISA_USE_POPCNT + #define MARISA_USE_POPCNT + #endif // MARISA_USE_POPCNT +#endif // MARISA_USE_SSE4_2 + +#ifdef MARISA_USE_SSE4_1 + #ifndef MARISA_USE_SSSE3 + #define MARISA_USE_SSSE3 + #endif // MARISA_USE_SSSE3 +#endif // MARISA_USE_SSE4_1 + +#ifdef MARISA_USE_POPCNT + #ifndef MARISA_USE_SSE3 + #define MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifdef _MSC_VER + #include <intrin.h> + #else // _MSC_VER + #include <popcntintrin.h> + #endif // _MSC_VER +#endif // MARISA_USE_POPCNT + +#ifdef MARISA_USE_SSSE3 + #ifndef MARISA_USE_SSE3 + #define MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifdef MARISA_X64 + #define MARISA_X64_SSSE3 + #else // MARISA_X64 + #define MARISA_X86_SSSE3 + #endif // MAIRSA_X64 + #include <tmmintrin.h> +#endif // MARISA_USE_SSSE3 + +#ifdef MARISA_USE_SSE3 + #ifndef MARISA_USE_SSE2 + #define MARISA_USE_SSE2 + #endif // MARISA_USE_SSE2 +#endif // MARISA_USE_SSE3 + +#ifdef MARISA_USE_SSE2 + #ifdef MARISA_X64 + #define MARISA_X64_SSE2 + #else // MARISA_X64 + #define MARISA_X86_SSE2 + #endif // MAIRSA_X64 + #include <emmintrin.h> +#endif // MARISA_USE_SSE2 + +#ifdef _MSC_VER + #if MARISA_WORD_SIZE == 64 + #include <intrin.h> + #pragma intrinsic(_BitScanForward64) + #else // MARISA_WORD_SIZE == 64 + #include <intrin.h> + #pragma intrinsic(_BitScanForward) + #endif // MARISA_WORD_SIZE == 64 +#endif // _MSC_VER + +#endif // MARISA_GRIMOIRE_INTRIN_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io.h new file mode 100644 index 00000000000..43ff8c91cf2 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io.h @@ -0,0 +1,18 @@ +#ifndef MARISA_GRIMOIRE_IO_H_ +#define MARISA_GRIMOIRE_IO_H_ + +#include "marisa/grimoire/io/mapper.h" +#include "marisa/grimoire/io/reader.h" +#include "marisa/grimoire/io/writer.h" + +namespace marisa { +namespace grimoire { + +using io::Mapper; +using io::Reader; +using io::Writer; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/mapper.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/mapper.cc new file mode 100644 index 00000000000..6ae0ee37f39 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/mapper.cc @@ -0,0 +1,163 @@ +#if (defined _WIN32) || (defined _WIN64) + #include <sys/types.h> + #include <sys/stat.h> + #include <windows.h> +#else // (defined _WIN32) || (defined _WIN64) + #include <sys/mman.h> + #include <sys/stat.h> + #include <sys/types.h> + #include <fcntl.h> + #include <unistd.h> +#endif // (defined _WIN32) || (defined _WIN64) + +#include "marisa/grimoire/io/mapper.h" + +namespace marisa { +namespace grimoire { +namespace io { + +#if (defined _WIN32) || (defined _WIN64) +Mapper::Mapper() + : ptr_(NULL), origin_(NULL), avail_(0), size_(0), + file_(NULL), map_(NULL) {} +#else // (defined _WIN32) || (defined _WIN64) +Mapper::Mapper() + : ptr_(NULL), origin_(MAP_FAILED), avail_(0), size_(0), fd_(-1) {} +#endif // (defined _WIN32) || (defined _WIN64) + +#if (defined _WIN32) || (defined _WIN64) +Mapper::~Mapper() { + if (origin_ != NULL) { + ::UnmapViewOfFile(origin_); + } + + if (map_ != NULL) { + ::CloseHandle(map_); + } + + if (file_ != NULL) { + ::CloseHandle(file_); + } +} +#else // (defined _WIN32) || (defined _WIN64) +Mapper::~Mapper() { + if (origin_ != MAP_FAILED) { + ::munmap(origin_, size_); + } + + if (fd_ != -1) { + ::close(fd_); + } +} +#endif // (defined _WIN32) || (defined _WIN64) + +void Mapper::open(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + Mapper temp; + temp.open_(filename); + swap(temp); +} + +void Mapper::open(const void *ptr, std::size_t size) { + MARISA_THROW_IF((ptr == NULL) && (size != 0), MARISA_NULL_ERROR); + + Mapper temp; + temp.open_(ptr, size); + swap(temp); +} + +void Mapper::seek(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + MARISA_THROW_IF(size > avail_, MARISA_IO_ERROR); + + map_data(size); +} + +bool Mapper::is_open() const { + return ptr_ != NULL; +} + +void Mapper::clear() { + Mapper().swap(*this); +} + +void Mapper::swap(Mapper &rhs) { + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(avail_, rhs.avail_); + marisa::swap(origin_, rhs.origin_); + marisa::swap(size_, rhs.size_); +#if (defined _WIN32) || (defined _WIN64) + marisa::swap(file_, rhs.file_); + marisa::swap(map_, rhs.map_); +#else // (defined _WIN32) || (defined _WIN64) + marisa::swap(fd_, rhs.fd_); +#endif // (defined _WIN32) || (defined _WIN64) +} + +const void *Mapper::map_data(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + MARISA_THROW_IF(size > avail_, MARISA_IO_ERROR); + + const char * const data = static_cast<const char *>(ptr_); + ptr_ = data + size; + avail_ -= size; + return data; +} + +#if (defined _WIN32) || (defined _WIN64) + #ifdef __MSVCRT_VERSION__ + #if __MSVCRT_VERSION__ >= 0x0601 + #define MARISA_HAS_STAT64 + #endif // __MSVCRT_VERSION__ >= 0x0601 + #endif // __MSVCRT_VERSION__ +void Mapper::open_(const char *filename) { + #ifdef MARISA_HAS_STAT64 + struct __stat64 st; + MARISA_THROW_IF(::_stat64(filename, &st) != 0, MARISA_IO_ERROR); + #else // MARISA_HAS_STAT64 + struct _stat st; + MARISA_THROW_IF(::_stat(filename, &st) != 0, MARISA_IO_ERROR); + #endif // MARISA_HAS_STAT64 + MARISA_THROW_IF((UInt64)st.st_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)st.st_size; + + file_ = ::CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + MARISA_THROW_IF(file_ == INVALID_HANDLE_VALUE, MARISA_IO_ERROR); + + map_ = ::CreateFileMapping(file_, NULL, PAGE_READONLY, 0, 0, NULL); + MARISA_THROW_IF(map_ == NULL, MARISA_IO_ERROR); + + origin_ = ::MapViewOfFile(map_, FILE_MAP_READ, 0, 0, 0); + MARISA_THROW_IF(origin_ == NULL, MARISA_IO_ERROR); + + ptr_ = static_cast<const char *>(origin_); + avail_ = size_; +} +#else // (defined _WIN32) || (defined _WIN64) +void Mapper::open_(const char *filename) { + struct stat st; + MARISA_THROW_IF(::stat(filename, &st) != 0, MARISA_IO_ERROR); + MARISA_THROW_IF((UInt64)st.st_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)st.st_size; + + fd_ = ::open(filename, O_RDONLY); + MARISA_THROW_IF(fd_ == -1, MARISA_IO_ERROR); + + origin_ = ::mmap(NULL, size_, PROT_READ, MAP_SHARED, fd_, 0); + MARISA_THROW_IF(origin_ == MAP_FAILED, MARISA_IO_ERROR); + + ptr_ = static_cast<const char *>(origin_); + avail_ = size_; +} +#endif // (defined _WIN32) || (defined _WIN64) + +void Mapper::open_(const void *ptr, std::size_t size) { + ptr_ = ptr; + avail_ = size; +} + +} // namespace io +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/mapper.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/mapper.h new file mode 100644 index 00000000000..f701c9477a6 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/mapper.h @@ -0,0 +1,67 @@ +#ifndef MARISA_GRIMOIRE_IO_MAPPER_H_ +#define MARISA_GRIMOIRE_IO_MAPPER_H_ + +#include <cstdio> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace io { + +class Mapper { + public: + Mapper(); + ~Mapper(); + + void open(const char *filename); + void open(const void *ptr, std::size_t size); + + template <typename T> + void map(T *obj) { + MARISA_THROW_IF(obj == NULL, MARISA_NULL_ERROR); + *obj = *static_cast<const T *>(map_data(sizeof(T))); + } + + template <typename T> + void map(const T **objs, std::size_t num_objs) { + MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)), + MARISA_SIZE_ERROR); + *objs = static_cast<const T *>(map_data(sizeof(T) * num_objs)); + } + + void seek(std::size_t size); + + bool is_open() const; + + void clear(); + void swap(Mapper &rhs); + + private: + const void *ptr_; + void *origin_; + std::size_t avail_; + std::size_t size_; +#if (defined _WIN32) || (defined _WIN64) + void *file_; + void *map_; +#else // (defined _WIN32) || (defined _WIN64) + int fd_; +#endif // (defined _WIN32) || (defined _WIN64) + + void open_(const char *filename); + void open_(const void *ptr, std::size_t size); + + const void *map_data(std::size_t size); + + // Disallows copy and assignment. + Mapper(const Mapper &); + Mapper &operator=(const Mapper &); +}; + +} // namespace io +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_MAPPER_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/reader.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/reader.cc new file mode 100644 index 00000000000..b9c0809886a --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/reader.cc @@ -0,0 +1,147 @@ +#include <stdio.h> + +#ifdef _WIN32 + #include <io.h> +#else // _WIN32 + #include <unistd.h> +#endif // _WIN32 + +#include <limits> + +#include "marisa/grimoire/io/reader.h" + +namespace marisa { +namespace grimoire { +namespace io { + +Reader::Reader() + : file_(NULL), fd_(-1), stream_(NULL), needs_fclose_(false) {} + +Reader::~Reader() { + if (needs_fclose_) { + ::fclose(file_); + } +} + +void Reader::open(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + Reader temp; + temp.open_(filename); + swap(temp); +} + +void Reader::open(std::FILE *file) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + + Reader temp; + temp.open_(file); + swap(temp); +} + +void Reader::open(int fd) { + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + Reader temp; + temp.open_(fd); + swap(temp); +} + +void Reader::open(std::istream &stream) { + Reader temp; + temp.open_(stream); + swap(temp); +} + +void Reader::clear() { + Reader().swap(*this); +} + +void Reader::swap(Reader &rhs) { + marisa::swap(file_, rhs.file_); + marisa::swap(fd_, rhs.fd_); + marisa::swap(stream_, rhs.stream_); + marisa::swap(needs_fclose_, rhs.needs_fclose_); +} + +void Reader::seek(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (size <= 16) { + char buf[16]; + read_data(buf, size); + } else { + char buf[1024]; + while (size != 0) { + const std::size_t count = (size < sizeof(buf)) ? size : sizeof(buf); + read_data(buf, count); + size -= count; + } + } +} + +bool Reader::is_open() const { + return (file_ != NULL) || (fd_ != -1) || (stream_ != NULL); +} + +void Reader::open_(const char *filename) { + std::FILE *file = NULL; +#ifdef _MSC_VER + MARISA_THROW_IF(::fopen_s(&file, filename, "rb") != 0, MARISA_IO_ERROR); +#else // _MSC_VER + file = ::fopen(filename, "rb"); + MARISA_THROW_IF(file == NULL, MARISA_IO_ERROR); +#endif // _MSC_VER + file_ = file; + needs_fclose_ = true; +} + +void Reader::open_(std::FILE *file) { + file_ = file; +} + +void Reader::open_(int fd) { + fd_ = fd; +} + +void Reader::open_(std::istream &stream) { + stream_ = &stream; +} + +void Reader::read_data(void *buf, std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (fd_ != -1) { + while (size != 0) { +#ifdef _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits<int>::max(); + const unsigned int count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const int size_read = ::_read(fd_, buf, count); +#else // _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits< ::ssize_t>::max(); + const ::size_t count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const ::ssize_t size_read = ::read(fd_, buf, count); +#endif // _WIN32 + MARISA_THROW_IF(size_read <= 0, MARISA_IO_ERROR); + buf = static_cast<char *>(buf) + size_read; + size -= static_cast<std::size_t>(size_read); + } + } else if (file_ != NULL) { + MARISA_THROW_IF(::fread(buf, 1, size, file_) != size, MARISA_IO_ERROR); + } else if (stream_ != NULL) { + try { + MARISA_THROW_IF(!stream_->read(static_cast<char *>(buf), + static_cast<std::streamsize>(size)), MARISA_IO_ERROR); + } catch (const std::ios_base::failure &) { + MARISA_THROW(MARISA_IO_ERROR, "std::ios_base::failure"); + } + } +} + +} // namespace io +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/reader.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/reader.h new file mode 100644 index 00000000000..fa4ce362782 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/reader.h @@ -0,0 +1,66 @@ +#ifndef MARISA_GRIMOIRE_IO_READER_H_ +#define MARISA_GRIMOIRE_IO_READER_H_ + +#include <cstdio> +#include <iostream> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace io { + +class Reader { + public: + Reader(); + ~Reader(); + + void open(const char *filename); + void open(std::FILE *file); + void open(int fd); + void open(std::istream &stream); + + template <typename T> + void read(T *obj) { + MARISA_THROW_IF(obj == NULL, MARISA_NULL_ERROR); + read_data(obj, sizeof(T)); + } + + template <typename T> + void read(T *objs, std::size_t num_objs) { + MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)), + MARISA_SIZE_ERROR); + read_data(objs, sizeof(T) * num_objs); + } + + void seek(std::size_t size); + + bool is_open() const; + + void clear(); + void swap(Reader &rhs); + + private: + std::FILE *file_; + int fd_; + std::istream *stream_; + bool needs_fclose_; + + void open_(const char *filename); + void open_(std::FILE *file); + void open_(int fd); + void open_(std::istream &stream); + + void read_data(void *buf, std::size_t size); + + // Disallows copy and assignment. + Reader(const Reader &); + Reader &operator=(const Reader &); +}; + +} // namespace io +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_READER_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/writer.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/writer.cc new file mode 100644 index 00000000000..fb3d2d08ff9 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/writer.cc @@ -0,0 +1,148 @@ +#include <stdio.h> + +#ifdef _WIN32 + #include <io.h> +#else // _WIN32 + #include <unistd.h> +#endif // _WIN32 + +#include <limits> + +#include "marisa/grimoire/io/writer.h" + +namespace marisa { +namespace grimoire { +namespace io { + +Writer::Writer() + : file_(NULL), fd_(-1), stream_(NULL), needs_fclose_(false) {} + +Writer::~Writer() { + if (needs_fclose_) { + ::fclose(file_); + } +} + +void Writer::open(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + Writer temp; + temp.open_(filename); + swap(temp); +} + +void Writer::open(std::FILE *file) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + + Writer temp; + temp.open_(file); + swap(temp); +} + +void Writer::open(int fd) { + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + Writer temp; + temp.open_(fd); + swap(temp); +} + +void Writer::open(std::ostream &stream) { + Writer temp; + temp.open_(stream); + swap(temp); +} + +void Writer::clear() { + Writer().swap(*this); +} + +void Writer::swap(Writer &rhs) { + marisa::swap(file_, rhs.file_); + marisa::swap(fd_, rhs.fd_); + marisa::swap(stream_, rhs.stream_); + marisa::swap(needs_fclose_, rhs.needs_fclose_); +} + +void Writer::seek(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (size <= 16) { + const char buf[16] = {}; + write_data(buf, size); + } else { + const char buf[1024] = {}; + do { + const std::size_t count = (size < sizeof(buf)) ? size : sizeof(buf); + write_data(buf, count); + size -= count; + } while (size != 0); + } +} + +bool Writer::is_open() const { + return (file_ != NULL) || (fd_ != -1) || (stream_ != NULL); +} + +void Writer::open_(const char *filename) { + std::FILE *file = NULL; +#ifdef _MSC_VER + MARISA_THROW_IF(::fopen_s(&file, filename, "wb") != 0, MARISA_IO_ERROR); +#else // _MSC_VER + file = ::fopen(filename, "wb"); + MARISA_THROW_IF(file == NULL, MARISA_IO_ERROR); +#endif // _MSC_VER + file_ = file; + needs_fclose_ = true; +} + +void Writer::open_(std::FILE *file) { + file_ = file; +} + +void Writer::open_(int fd) { + fd_ = fd; +} + +void Writer::open_(std::ostream &stream) { + stream_ = &stream; +} + +void Writer::write_data(const void *data, std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (fd_ != -1) { + while (size != 0) { +#ifdef _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits<int>::max(); + const unsigned int count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const int size_written = ::_write(fd_, data, count); +#else // _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits< ::ssize_t>::max(); + const ::size_t count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const ::ssize_t size_written = ::write(fd_, data, count); +#endif // _WIN32 + MARISA_THROW_IF(size_written <= 0, MARISA_IO_ERROR); + data = static_cast<const char *>(data) + size_written; + size -= static_cast<std::size_t>(size_written); + } + } else if (file_ != NULL) { + MARISA_THROW_IF(::fwrite(data, 1, size, file_) != size, MARISA_IO_ERROR); + MARISA_THROW_IF(::fflush(file_) != 0, MARISA_IO_ERROR); + } else if (stream_ != NULL) { + try { + MARISA_THROW_IF(!stream_->write(static_cast<const char *>(data), + static_cast<std::streamsize>(size)), MARISA_IO_ERROR); + } catch (const std::ios_base::failure &) { + MARISA_THROW(MARISA_IO_ERROR, "std::ios_base::failure"); + } + } +} + +} // namespace io +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/writer.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/writer.h new file mode 100644 index 00000000000..d49761b0227 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/io/writer.h @@ -0,0 +1,65 @@ +#ifndef MARISA_GRIMOIRE_IO_WRITER_H_ +#define MARISA_GRIMOIRE_IO_WRITER_H_ + +#include <cstdio> +#include <iostream> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace io { + +class Writer { + public: + Writer(); + ~Writer(); + + void open(const char *filename); + void open(std::FILE *file); + void open(int fd); + void open(std::ostream &stream); + + template <typename T> + void write(const T &obj) { + write_data(&obj, sizeof(T)); + } + + template <typename T> + void write(const T *objs, std::size_t num_objs) { + MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)), + MARISA_SIZE_ERROR); + write_data(objs, sizeof(T) * num_objs); + } + + void seek(std::size_t size); + + bool is_open() const; + + void clear(); + void swap(Writer &rhs); + + private: + std::FILE *file_; + int fd_; + std::ostream *stream_; + bool needs_fclose_; + + void open_(const char *filename); + void open_(std::FILE *file); + void open_(int fd); + void open_(std::ostream &stream); + + void write_data(const void *data, std::size_t size); + + // Disallows copy and assignment. + Writer(const Writer &); + Writer &operator=(const Writer &); +}; + +} // namespace io +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_WRITER_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie.h new file mode 100644 index 00000000000..73a0c2bf7aa --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie.h @@ -0,0 +1,16 @@ +#ifndef MARISA_GRIMOIRE_TRIE_H_ +#define MARISA_GRIMOIRE_TRIE_H_ + +#include "marisa/grimoire/trie/state.h" +#include "marisa/grimoire/trie/louds-trie.h" + +namespace marisa { +namespace grimoire { + +using trie::State; +using trie::LoudsTrie; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/cache.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/cache.h new file mode 100644 index 00000000000..19ce021c32d --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/cache.h @@ -0,0 +1,81 @@ +#ifndef MARISA_GRIMOIRE_TRIE_CACHE_H_ +#define MARISA_GRIMOIRE_TRIE_CACHE_H_ + +#include <cfloat> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Cache { + public: + Cache() : parent_(0), child_(0), union_() { + union_.weight = FLT_MIN; + } + Cache(const Cache &cache) + : parent_(cache.parent_), child_(cache.child_), union_(cache.union_) {} + + Cache &operator=(const Cache &cache) { + parent_ = cache.parent_; + child_ = cache.child_; + union_ = cache.union_; + return *this; + } + + void set_parent(std::size_t parent) { + MARISA_DEBUG_IF(parent > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + parent_ = (UInt32)parent; + } + void set_child(std::size_t child) { + MARISA_DEBUG_IF(child > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + child_ = (UInt32)child; + } + void set_base(UInt8 base) { + union_.link = (union_.link & ~0xFFU) | base; + } + void set_extra(std::size_t extra) { + MARISA_DEBUG_IF(extra > (MARISA_UINT32_MAX >> 8), MARISA_SIZE_ERROR); + union_.link = (UInt32)((union_.link & 0xFFU) | (extra << 8)); + } + void set_weight(float weight) { + union_.weight = weight; + } + + std::size_t parent() const { + return parent_; + } + std::size_t child() const { + return child_; + } + UInt8 base() const { + return (UInt8)(union_.link & 0xFFU); + } + std::size_t extra() const { + return union_.link >> 8; + } + char label() const { + return (char)base(); + } + std::size_t link() const { + return union_.link; + } + float weight() const { + return union_.weight; + } + + private: + UInt32 parent_; + UInt32 child_; + union Union { + UInt32 link; + float weight; + } union_; +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_CACHE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/config.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/config.h new file mode 100644 index 00000000000..2f1e17a0643 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/config.h @@ -0,0 +1,155 @@ +#ifndef MARISA_GRIMOIRE_TRIE_CONFIG_H_ +#define MARISA_GRIMOIRE_TRIE_CONFIG_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Config { + public: + Config() + : num_tries_(MARISA_DEFAULT_NUM_TRIES), + cache_level_(MARISA_DEFAULT_CACHE), + tail_mode_(MARISA_DEFAULT_TAIL), + node_order_(MARISA_DEFAULT_ORDER) {} + + void parse(int config_flags) { + Config temp; + temp.parse_(config_flags); + swap(temp); + } + + int flags() const { + return (int)num_tries_ | tail_mode_ | node_order_; + } + + std::size_t num_tries() const { + return num_tries_; + } + CacheLevel cache_level() const { + return cache_level_; + } + TailMode tail_mode() const { + return tail_mode_; + } + NodeOrder node_order() const { + return node_order_; + } + + void clear() { + Config().swap(*this); + } + void swap(Config &rhs) { + marisa::swap(num_tries_, rhs.num_tries_); + marisa::swap(cache_level_, rhs.cache_level_); + marisa::swap(tail_mode_, rhs.tail_mode_); + marisa::swap(node_order_, rhs.node_order_); + } + + private: + std::size_t num_tries_; + CacheLevel cache_level_; + TailMode tail_mode_; + NodeOrder node_order_; + + void parse_(int config_flags) { + MARISA_THROW_IF((config_flags & ~MARISA_CONFIG_MASK) != 0, + MARISA_CODE_ERROR); + + parse_num_tries(config_flags); + parse_cache_level(config_flags); + parse_tail_mode(config_flags); + parse_node_order(config_flags); + } + + void parse_num_tries(int config_flags) { + const int num_tries = config_flags & MARISA_NUM_TRIES_MASK; + if (num_tries != 0) { + num_tries_ = static_cast<std::size_t>(num_tries); + } + } + + void parse_cache_level(int config_flags) { + switch (config_flags & MARISA_CACHE_LEVEL_MASK) { + case 0: { + cache_level_ = MARISA_DEFAULT_CACHE; + break; + } + case MARISA_HUGE_CACHE: { + cache_level_ = MARISA_HUGE_CACHE; + break; + } + case MARISA_LARGE_CACHE: { + cache_level_ = MARISA_LARGE_CACHE; + break; + } + case MARISA_NORMAL_CACHE: { + cache_level_ = MARISA_NORMAL_CACHE; + break; + } + case MARISA_SMALL_CACHE: { + cache_level_ = MARISA_SMALL_CACHE; + break; + } + case MARISA_TINY_CACHE: { + cache_level_ = MARISA_TINY_CACHE; + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined cache level"); + } + } + } + + void parse_tail_mode(int config_flags) { + switch (config_flags & MARISA_TAIL_MODE_MASK) { + case 0: { + tail_mode_ = MARISA_DEFAULT_TAIL; + break; + } + case MARISA_TEXT_TAIL: { + tail_mode_ = MARISA_TEXT_TAIL; + break; + } + case MARISA_BINARY_TAIL: { + tail_mode_ = MARISA_BINARY_TAIL; + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined tail mode"); + } + } + } + + void parse_node_order(int config_flags) { + switch (config_flags & MARISA_NODE_ORDER_MASK) { + case 0: { + node_order_ = MARISA_DEFAULT_ORDER; + break; + } + case MARISA_LABEL_ORDER: { + node_order_ = MARISA_LABEL_ORDER; + break; + } + case MARISA_WEIGHT_ORDER: { + node_order_ = MARISA_WEIGHT_ORDER; + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined node order"); + } + } + } + + // Disallows copy and assignment. + Config(const Config &); + Config &operator=(const Config &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_CONFIG_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/entry.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/entry.h new file mode 100644 index 00000000000..2c754a85335 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/entry.h @@ -0,0 +1,81 @@ +#ifndef MARISA_GRIMOIRE_TRIE_ENTRY_H_ +#define MARISA_GRIMOIRE_TRIE_ENTRY_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Entry { + public: + Entry() : ptr_(NULL), length_(0), id_(0) {} + Entry(const Entry &entry) + : ptr_(entry.ptr_), length_(entry.length_), id_(entry.id_) {} + + Entry &operator=(const Entry &entry) { + ptr_ = entry.ptr_; + length_ = entry.length_; + id_ = entry.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return *(ptr_ - i); + } + + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr + length - 1; + length_ = (UInt32)length; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + id_ = (UInt32)id; + } + + const char *ptr() const { + return ptr_ - length_ + 1; + } + std::size_t length() const { + return length_; + } + std::size_t id() const { + return id_; + } + + class StringComparer { + public: + bool operator()(const Entry &lhs, const Entry &rhs) const { + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return true; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] > (UInt8)rhs[i]; + } + } + return lhs.length() > rhs.length(); + } + }; + + class IDComparer { + public: + bool operator()(const Entry &lhs, const Entry &rhs) const { + return lhs.id_ < rhs.id_; + } + }; + + private: + const char *ptr_; + UInt32 length_; + UInt32 id_; +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_ENTRY_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/header.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/header.h new file mode 100644 index 00000000000..e13220c6a99 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/header.h @@ -0,0 +1,61 @@ +#ifndef MARISA_GRIMOIRE_TRIE_HEADER_H_ +#define MARISA_GRIMOIRE_TRIE_HEADER_H_ + +#include "marisa/grimoire/io.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Header { + public: + enum { + HEADER_SIZE = 16 + }; + + Header() {} + + void map(Mapper &mapper) { + const char *ptr; + mapper.map(&ptr, HEADER_SIZE); + MARISA_THROW_IF(!test_header(ptr), MARISA_FORMAT_ERROR); + } + void read(Reader &reader) { + char buf[HEADER_SIZE]; + reader.read(buf, HEADER_SIZE); + MARISA_THROW_IF(!test_header(buf), MARISA_FORMAT_ERROR); + } + void write(Writer &writer) const { + writer.write(get_header(), HEADER_SIZE); + } + + std::size_t io_size() const { + return HEADER_SIZE; + } + + private: + + static const char *get_header() { + static const char buf[HEADER_SIZE] = "We love Marisa."; + return buf; + } + + static bool test_header(const char *ptr) { + for (std::size_t i = 0; i < HEADER_SIZE; ++i) { + if (ptr[i] != get_header()[i]) { + return false; + } + } + return true; + } + + // Disallows copy and assignment. + Header(const Header &); + Header &operator=(const Header &); +}; + +} // namespace trie +} // namespace marisa +} // namespace grimoire + +#endif // MARISA_GRIMOIRE_TRIE_HEADER_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/history.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/history.h new file mode 100644 index 00000000000..84d10df1f19 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/history.h @@ -0,0 +1,65 @@ +#ifndef MARISA_GRIMOIRE_TRIE_STATE_HISTORY_H_ +#define MARISA_GRIMOIRE_TRIE_STATE_HISTORY_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class History { + public: + History() + : node_id_(0), louds_pos_(0), key_pos_(0), + link_id_(MARISA_INVALID_LINK_ID), key_id_(MARISA_INVALID_KEY_ID) {} + + void set_node_id(std::size_t node_id) { + MARISA_DEBUG_IF(node_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + node_id_ = (UInt32)node_id; + } + void set_louds_pos(std::size_t louds_pos) { + MARISA_DEBUG_IF(louds_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + louds_pos_ = (UInt32)louds_pos; + } + void set_key_pos(std::size_t key_pos) { + MARISA_DEBUG_IF(key_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_pos_ = (UInt32)key_pos; + } + void set_link_id(std::size_t link_id) { + MARISA_DEBUG_IF(link_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + link_id_ = (UInt32)link_id; + } + void set_key_id(std::size_t key_id) { + MARISA_DEBUG_IF(key_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_id_ = (UInt32)key_id; + } + + std::size_t node_id() const { + return node_id_; + } + std::size_t louds_pos() const { + return louds_pos_; + } + std::size_t key_pos() const { + return key_pos_; + } + std::size_t link_id() const { + return link_id_; + } + std::size_t key_id() const { + return key_id_; + } + + private: + UInt32 node_id_; + UInt32 louds_pos_; + UInt32 key_pos_; + UInt32 link_id_; + UInt32 key_id_; +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_STATE_HISTORY_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/key.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/key.h new file mode 100644 index 00000000000..8555cc72d2c --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/key.h @@ -0,0 +1,226 @@ +#ifndef MARISA_GRIMOIRE_TRIE_KEY_H_ +#define MARISA_GRIMOIRE_TRIE_KEY_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Key { + public: + Key() : ptr_(NULL), length_(0), union_(), id_(0) { + union_.terminal = 0; + } + Key(const Key &entry) + : ptr_(entry.ptr_), length_(entry.length_), + union_(entry.union_), id_(entry.id_) {} + + Key &operator=(const Key &entry) { + ptr_ = entry.ptr_; + length_ = entry.length_; + union_ = entry.union_; + id_ = entry.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return ptr_[i]; + } + + void substr(std::size_t pos, std::size_t length) { + MARISA_DEBUG_IF(pos > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(length > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(pos > (length_ - length), MARISA_BOUND_ERROR); + ptr_ += pos; + length_ = (UInt32)length; + } + + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr; + length_ = (UInt32)length; + } + void set_weight(float weight) { + union_.weight = weight; + } + void set_terminal(std::size_t terminal) { + MARISA_DEBUG_IF(terminal > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + union_.terminal = (UInt32)terminal; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + id_ = (UInt32)id; + } + + const char *ptr() const { + return ptr_; + } + std::size_t length() const { + return length_; + } + float weight() const { + return union_.weight; + } + std::size_t terminal() const { + return union_.terminal; + } + std::size_t id() const { + return id_; + } + + private: + const char *ptr_; + UInt32 length_; + union Union { + float weight; + UInt32 terminal; + } union_; + UInt32 id_; +}; + +inline bool operator==(const Key &lhs, const Key &rhs) { + if (lhs.length() != rhs.length()) { + return false; + } + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + +inline bool operator!=(const Key &lhs, const Key &rhs) { + return !(lhs == rhs); +} + +inline bool operator<(const Key &lhs, const Key &rhs) { + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return false; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] < (UInt8)rhs[i]; + } + } + return lhs.length() < rhs.length(); +} + +inline bool operator>(const Key &lhs, const Key &rhs) { + return rhs < lhs; +} + +class ReverseKey { + public: + ReverseKey() : ptr_(NULL), length_(0), union_(), id_(0) { + union_.terminal = 0; + } + ReverseKey(const ReverseKey &entry) + : ptr_(entry.ptr_), length_(entry.length_), + union_(entry.union_), id_(entry.id_) {} + + ReverseKey &operator=(const ReverseKey &entry) { + ptr_ = entry.ptr_; + length_ = entry.length_; + union_ = entry.union_; + id_ = entry.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return *(ptr_ - i - 1); + } + + void substr(std::size_t pos, std::size_t length) { + MARISA_DEBUG_IF(pos > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(length > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(pos > (length_ - length), MARISA_BOUND_ERROR); + ptr_ -= pos; + length_ = (UInt32)length; + } + + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr + length; + length_ = (UInt32)length; + } + void set_weight(float weight) { + union_.weight = weight; + } + void set_terminal(std::size_t terminal) { + MARISA_DEBUG_IF(terminal > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + union_.terminal = (UInt32)terminal; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + id_ = (UInt32)id; + } + + const char *ptr() const { + return ptr_ - length_; + } + std::size_t length() const { + return length_; + } + float weight() const { + return union_.weight; + } + std::size_t terminal() const { + return union_.terminal; + } + std::size_t id() const { + return id_; + } + + private: + const char *ptr_; + UInt32 length_; + union Union { + float weight; + UInt32 terminal; + } union_; + UInt32 id_; +}; + +inline bool operator==(const ReverseKey &lhs, const ReverseKey &rhs) { + if (lhs.length() != rhs.length()) { + return false; + } + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + +inline bool operator!=(const ReverseKey &lhs, const ReverseKey &rhs) { + return !(lhs == rhs); +} + +inline bool operator<(const ReverseKey &lhs, const ReverseKey &rhs) { + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return false; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] < (UInt8)rhs[i]; + } + } + return lhs.length() < rhs.length(); +} + +inline bool operator>(const ReverseKey &lhs, const ReverseKey &rhs) { + return rhs < lhs; +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_KEY_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc new file mode 100644 index 00000000000..df191f5191e --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc @@ -0,0 +1,878 @@ +#include <algorithm> +#include <functional> +#include <queue> + +#include "marisa/grimoire/algorithm.h" +#include "marisa/grimoire/trie/header.h" +#include "marisa/grimoire/trie/range.h" +#include "marisa/grimoire/trie/state.h" +#include "marisa/grimoire/trie/louds-trie.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +LoudsTrie::LoudsTrie() + : louds_(), terminal_flags_(), link_flags_(), bases_(), extras_(), + tail_(), next_trie_(), cache_(), cache_mask_(0), num_l1_nodes_(0), + config_(), mapper_() {} + +LoudsTrie::~LoudsTrie() {} + +void LoudsTrie::build(Keyset &keyset, int flags) { + Config config; + config.parse(flags); + + LoudsTrie temp; + temp.build_(keyset, config); + swap(temp); +} + +void LoudsTrie::map(Mapper &mapper) { + Header().map(mapper); + + LoudsTrie temp; + temp.map_(mapper); + temp.mapper_.swap(mapper); + swap(temp); +} + +void LoudsTrie::read(Reader &reader) { + Header().read(reader); + + LoudsTrie temp; + temp.read_(reader); + swap(temp); +} + +void LoudsTrie::write(Writer &writer) const { + Header().write(writer); + + write_(writer); +} + +bool LoudsTrie::lookup(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + + State &state = agent.state(); + state.lookup_init(); + while (state.query_pos() < agent.query().length()) { + if (!find_child(agent)) { + return false; + } + } + if (!terminal_flags_[state.node_id()]) { + return false; + } + agent.set_key(agent.query().ptr(), agent.query().length()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; +} + +void LoudsTrie::reverse_lookup(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + MARISA_THROW_IF(agent.query().id() >= size(), MARISA_BOUND_ERROR); + + State &state = agent.state(); + state.reverse_lookup_init(); + + state.set_node_id(terminal_flags_.select1(agent.query().id())); + if (state.node_id() == 0) { + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(agent.query().id()); + return; + } + for ( ; ; ) { + if (link_flags_[state.node_id()]) { + const std::size_t prev_key_pos = state.key_buf().size(); + restore(agent, get_link(state.node_id())); + std::reverse(state.key_buf().begin() + prev_key_pos, + state.key_buf().end()); + } else { + state.key_buf().push_back((char)bases_[state.node_id()]); + } + + if (state.node_id() <= num_l1_nodes_) { + std::reverse(state.key_buf().begin(), state.key_buf().end()); + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(agent.query().id()); + return; + } + state.set_node_id(louds_.select1(state.node_id()) - state.node_id() - 1); + } +} + +bool LoudsTrie::common_prefix_search(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (state.status_code() == MARISA_END_OF_COMMON_PREFIX_SEARCH) { + return false; + } + + if (state.status_code() != MARISA_READY_TO_COMMON_PREFIX_SEARCH) { + state.common_prefix_search_init(); + if (terminal_flags_[state.node_id()]) { + agent.set_key(agent.query().ptr(), state.query_pos()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; + } + } + + while (state.query_pos() < agent.query().length()) { + if (!find_child(agent)) { + state.set_status_code(MARISA_END_OF_COMMON_PREFIX_SEARCH); + return false; + } else if (terminal_flags_[state.node_id()]) { + agent.set_key(agent.query().ptr(), state.query_pos()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; + } + } + state.set_status_code(MARISA_END_OF_COMMON_PREFIX_SEARCH); + return false; +} + +bool LoudsTrie::predictive_search(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (state.status_code() == MARISA_END_OF_PREDICTIVE_SEARCH) { + return false; + } + + if (state.status_code() != MARISA_READY_TO_PREDICTIVE_SEARCH) { + state.predictive_search_init(); + while (state.query_pos() < agent.query().length()) { + if (!predictive_find_child(agent)) { + state.set_status_code(MARISA_END_OF_PREDICTIVE_SEARCH); + return false; + } + } + + History history; + history.set_node_id(state.node_id()); + history.set_key_pos(state.key_buf().size()); + state.history().push_back(history); + state.set_history_pos(1); + + if (terminal_flags_[state.node_id()]) { + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; + } + } + + for ( ; ; ) { + if (state.history_pos() == state.history().size()) { + const History ¤t = state.history().back(); + History next; + next.set_louds_pos(louds_.select0(current.node_id()) + 1); + next.set_node_id(next.louds_pos() - current.node_id() - 1); + state.history().push_back(next); + } + + History &next = state.history()[state.history_pos()]; + const bool link_flag = louds_[next.louds_pos()]; + next.set_louds_pos(next.louds_pos() + 1); + if (link_flag) { + state.set_history_pos(state.history_pos() + 1); + if (link_flags_[next.node_id()]) { + next.set_link_id(update_link_id(next.link_id(), next.node_id())); + restore(agent, get_link(next.node_id(), next.link_id())); + } else { + state.key_buf().push_back((char)bases_[next.node_id()]); + } + next.set_key_pos(state.key_buf().size()); + + if (terminal_flags_[next.node_id()]) { + if (next.key_id() == MARISA_INVALID_KEY_ID) { + next.set_key_id(terminal_flags_.rank1(next.node_id())); + } else { + next.set_key_id(next.key_id() + 1); + } + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(next.key_id()); + return true; + } + } else if (state.history_pos() != 1) { + History ¤t = state.history()[state.history_pos() - 1]; + current.set_node_id(current.node_id() + 1); + const History &prev = + state.history()[state.history_pos() - 2]; + state.key_buf().resize(prev.key_pos()); + state.set_history_pos(state.history_pos() - 1); + } else { + state.set_status_code(MARISA_END_OF_PREDICTIVE_SEARCH); + return false; + } + } +} + +std::size_t LoudsTrie::total_size() const { + return louds_.total_size() + terminal_flags_.total_size() + + link_flags_.total_size() + bases_.total_size() + + extras_.total_size() + tail_.total_size() + + ((next_trie_.get() != NULL) ? next_trie_->total_size() : 0) + + cache_.total_size(); +} + +std::size_t LoudsTrie::io_size() const { + return Header().io_size() + louds_.io_size() + + terminal_flags_.io_size() + link_flags_.io_size() + + bases_.io_size() + extras_.io_size() + tail_.io_size() + + ((next_trie_.get() != NULL) ? + (next_trie_->io_size() - Header().io_size()) : 0) + + cache_.io_size() + (sizeof(UInt32) * 2); +} + +void LoudsTrie::clear() { + LoudsTrie().swap(*this); +} + +void LoudsTrie::swap(LoudsTrie &rhs) { + louds_.swap(rhs.louds_); + terminal_flags_.swap(rhs.terminal_flags_); + link_flags_.swap(rhs.link_flags_); + bases_.swap(rhs.bases_); + extras_.swap(rhs.extras_); + tail_.swap(rhs.tail_); + next_trie_.swap(rhs.next_trie_); + cache_.swap(rhs.cache_); + marisa::swap(cache_mask_, rhs.cache_mask_); + marisa::swap(num_l1_nodes_, rhs.num_l1_nodes_); + config_.swap(rhs.config_); + mapper_.swap(rhs.mapper_); +} + +void LoudsTrie::build_(Keyset &keyset, const Config &config) { + Vector<Key> keys; + keys.resize(keyset.size()); + for (std::size_t i = 0; i < keyset.size(); ++i) { + keys[i].set_str(keyset[i].ptr(), keyset[i].length()); + keys[i].set_weight(keyset[i].weight()); + } + + Vector<UInt32> terminals; + build_trie(keys, &terminals, config, 1); + + typedef std::pair<UInt32, UInt32> TerminalIdPair; + + Vector<TerminalIdPair> pairs; + pairs.resize(terminals.size()); + for (std::size_t i = 0; i < pairs.size(); ++i) { + pairs[i].first = terminals[i]; + pairs[i].second = (UInt32)i; + } + terminals.clear(); + std::sort(pairs.begin(), pairs.end()); + + std::size_t node_id = 0; + for (std::size_t i = 0; i < pairs.size(); ++i) { + while (node_id < pairs[i].first) { + terminal_flags_.push_back(false); + ++node_id; + } + if (node_id == pairs[i].first) { + terminal_flags_.push_back(true); + ++node_id; + } + } + while (node_id < bases_.size()) { + terminal_flags_.push_back(false); + ++node_id; + } + terminal_flags_.push_back(false); + terminal_flags_.build(false, true); + + for (std::size_t i = 0; i < keyset.size(); ++i) { + keyset[pairs[i].second].set_id(terminal_flags_.rank1(pairs[i].first)); + } +} + +template <typename T> +void LoudsTrie::build_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id) { + build_current_trie(keys, terminals, config, trie_id); + + Vector<UInt32> next_terminals; + if (!keys.empty()) { + build_next_trie(keys, &next_terminals, config, trie_id); + } + + if (next_trie_.get() != NULL) { + config_.parse(static_cast<int>((next_trie_->num_tries() + 1)) | + next_trie_->tail_mode() | next_trie_->node_order()); + } else { + config_.parse(1 | tail_.mode() | config.node_order() | + config.cache_level()); + } + + link_flags_.build(false, false); + std::size_t node_id = 0; + for (std::size_t i = 0; i < next_terminals.size(); ++i) { + while (!link_flags_[node_id]) { + ++node_id; + } + bases_[node_id] = (UInt8)(next_terminals[i] % 256); + next_terminals[i] /= 256; + ++node_id; + } + extras_.build(next_terminals); + fill_cache(); +} + +template <typename T> +void LoudsTrie::build_current_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, + std::size_t trie_id) try { + for (std::size_t i = 0; i < keys.size(); ++i) { + keys[i].set_id(i); + } + const std::size_t num_keys = Algorithm().sort(keys.begin(), keys.end()); + reserve_cache(config, trie_id, num_keys); + + louds_.push_back(true); + louds_.push_back(false); + bases_.push_back('\0'); + link_flags_.push_back(false); + + Vector<T> next_keys; + std::queue<Range> queue; + Vector<WeightedRange> w_ranges; + + queue.push(make_range(0, keys.size(), 0)); + while (!queue.empty()) { + const std::size_t node_id = link_flags_.size() - queue.size(); + + Range range = queue.front(); + queue.pop(); + + while ((range.begin() < range.end()) && + (keys[range.begin()].length() == range.key_pos())) { + keys[range.begin()].set_terminal(node_id); + range.set_begin(range.begin() + 1); + } + + if (range.begin() == range.end()) { + louds_.push_back(false); + continue; + } + + w_ranges.clear(); + double weight = keys[range.begin()].weight(); + for (std::size_t i = range.begin() + 1; i < range.end(); ++i) { + if (keys[i - 1][range.key_pos()] != keys[i][range.key_pos()]) { + w_ranges.push_back(make_weighted_range( + range.begin(), i, range.key_pos(), (float)weight)); + range.set_begin(i); + weight = 0.0; + } + weight += keys[i].weight(); + } + w_ranges.push_back(make_weighted_range( + range.begin(), range.end(), range.key_pos(), (float)weight)); + if (config.node_order() == MARISA_WEIGHT_ORDER) { + std::stable_sort(w_ranges.begin(), w_ranges.end(), + std::greater<WeightedRange>()); + } + + if (node_id == 0) { + num_l1_nodes_ = w_ranges.size(); + } + + for (std::size_t i = 0; i < w_ranges.size(); ++i) { + WeightedRange &w_range = w_ranges[i]; + std::size_t key_pos = w_range.key_pos() + 1; + while (key_pos < keys[w_range.begin()].length()) { + std::size_t j; + for (j = w_range.begin() + 1; j < w_range.end(); ++j) { + if (keys[j - 1][key_pos] != keys[j][key_pos]) { + break; + } + } + if (j < w_range.end()) { + break; + } + ++key_pos; + } + cache<T>(node_id, bases_.size(), w_range.weight(), + keys[w_range.begin()][w_range.key_pos()]); + + if (key_pos == w_range.key_pos() + 1) { + bases_.push_back(static_cast<unsigned char>( + keys[w_range.begin()][w_range.key_pos()])); + link_flags_.push_back(false); + } else { + bases_.push_back('\0'); + link_flags_.push_back(true); + T next_key; + next_key.set_str(keys[w_range.begin()].ptr(), + keys[w_range.begin()].length()); + next_key.substr(w_range.key_pos(), key_pos - w_range.key_pos()); + next_key.set_weight(w_range.weight()); + next_keys.push_back(next_key); + } + w_range.set_key_pos(key_pos); + queue.push(w_range.range()); + louds_.push_back(true); + } + louds_.push_back(false); + } + + louds_.push_back(false); + louds_.build(trie_id == 1, true); + bases_.shrink(); + + build_terminals(keys, terminals); + keys.swap(next_keys); +} catch (const std::bad_alloc &) { + MARISA_THROW(MARISA_MEMORY_ERROR, "std::bad_alloc"); +} + +template <> +void LoudsTrie::build_next_trie(Vector<Key> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id) { + if (trie_id == config.num_tries()) { + Vector<Entry> entries; + entries.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + entries[i].set_str(keys[i].ptr(), keys[i].length()); + } + tail_.build(entries, terminals, config.tail_mode()); + return; + } + Vector<ReverseKey> reverse_keys; + reverse_keys.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + reverse_keys[i].set_str(keys[i].ptr(), keys[i].length()); + reverse_keys[i].set_weight(keys[i].weight()); + } + keys.clear(); + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->build_trie(reverse_keys, terminals, config, trie_id + 1); +} + +template <> +void LoudsTrie::build_next_trie(Vector<ReverseKey> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id) { + if (trie_id == config.num_tries()) { + Vector<Entry> entries; + entries.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + entries[i].set_str(keys[i].ptr(), keys[i].length()); + } + tail_.build(entries, terminals, config.tail_mode()); + return; + } + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->build_trie(keys, terminals, config, trie_id + 1); +} + +template <typename T> +void LoudsTrie::build_terminals(const Vector<T> &keys, + Vector<UInt32> *terminals) const { + Vector<UInt32> temp; + temp.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + temp[keys[i].id()] = (UInt32)keys[i].terminal(); + } + terminals->swap(temp); +} + +template <> +void LoudsTrie::cache<Key>(std::size_t parent, std::size_t child, + float weight, char label) { + MARISA_DEBUG_IF(parent >= child, MARISA_RANGE_ERROR); + + const std::size_t cache_id = get_cache_id(parent, label); + if (weight > cache_[cache_id].weight()) { + cache_[cache_id].set_parent(parent); + cache_[cache_id].set_child(child); + cache_[cache_id].set_weight(weight); + } +} + +void LoudsTrie::reserve_cache(const Config &config, std::size_t trie_id, + std::size_t num_keys) { + std::size_t cache_size = (trie_id == 1) ? 256 : 1; + while (cache_size < (num_keys / config.cache_level())) { + cache_size *= 2; + } + cache_.resize(cache_size); + cache_mask_ = cache_size - 1; +} + +template <> +void LoudsTrie::cache<ReverseKey>(std::size_t parent, std::size_t child, + float weight, char) { + MARISA_DEBUG_IF(parent >= child, MARISA_RANGE_ERROR); + + const std::size_t cache_id = get_cache_id(child); + if (weight > cache_[cache_id].weight()) { + cache_[cache_id].set_parent(parent); + cache_[cache_id].set_child(child); + cache_[cache_id].set_weight(weight); + } +} + +void LoudsTrie::fill_cache() { + for (std::size_t i = 0; i < cache_.size(); ++i) { + const std::size_t node_id = cache_[i].child(); + if (node_id != 0) { + cache_[i].set_base(bases_[node_id]); + cache_[i].set_extra(!link_flags_[node_id] ? + MARISA_INVALID_EXTRA : extras_[link_flags_.rank1(node_id)]); + } else { + cache_[i].set_parent(MARISA_UINT32_MAX); + cache_[i].set_child(MARISA_UINT32_MAX); + } + } +} + +void LoudsTrie::map_(Mapper &mapper) { + louds_.map(mapper); + terminal_flags_.map(mapper); + link_flags_.map(mapper); + bases_.map(mapper); + extras_.map(mapper); + tail_.map(mapper); + if ((link_flags_.num_1s() != 0) && tail_.empty()) { + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->map_(mapper); + } + cache_.map(mapper); + cache_mask_ = cache_.size() - 1; + { + UInt32 temp_num_l1_nodes; + mapper.map(&temp_num_l1_nodes); + num_l1_nodes_ = temp_num_l1_nodes; + } + { + UInt32 temp_config_flags; + mapper.map(&temp_config_flags); + config_.parse((int)temp_config_flags); + } +} + +void LoudsTrie::read_(Reader &reader) { + louds_.read(reader); + terminal_flags_.read(reader); + link_flags_.read(reader); + bases_.read(reader); + extras_.read(reader); + tail_.read(reader); + if ((link_flags_.num_1s() != 0) && tail_.empty()) { + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->read_(reader); + } + cache_.read(reader); + cache_mask_ = cache_.size() - 1; + { + UInt32 temp_num_l1_nodes; + reader.read(&temp_num_l1_nodes); + num_l1_nodes_ = temp_num_l1_nodes; + } + { + UInt32 temp_config_flags; + reader.read(&temp_config_flags); + config_.parse((int)temp_config_flags); + } +} + +void LoudsTrie::write_(Writer &writer) const { + louds_.write(writer); + terminal_flags_.write(writer); + link_flags_.write(writer); + bases_.write(writer); + extras_.write(writer); + tail_.write(writer); + if (next_trie_.get() != NULL) { + next_trie_->write_(writer); + } + cache_.write(writer); + writer.write((UInt32)num_l1_nodes_); + writer.write((UInt32)config_.flags()); +} + +bool LoudsTrie::find_child(Agent &agent) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + + State &state = agent.state(); + const std::size_t cache_id = get_cache_id(state.node_id(), + agent.query()[state.query_pos()]); + if (state.node_id() == cache_[cache_id].parent()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!match(agent, cache_[cache_id].link())) { + return false; + } + } else { + state.set_query_pos(state.query_pos() + 1); + } + state.set_node_id(cache_[cache_id].child()); + return true; + } + + std::size_t louds_pos = louds_.select0(state.node_id()) + 1; + if (!louds_[louds_pos]) { + return false; + } + state.set_node_id(louds_pos - state.node_id() - 1); + std::size_t link_id = MARISA_INVALID_LINK_ID; + do { + if (link_flags_[state.node_id()]) { + link_id = update_link_id(link_id, state.node_id()); + const std::size_t prev_query_pos = state.query_pos(); + if (match(agent, get_link(state.node_id(), link_id))) { + return true; + } else if (state.query_pos() != prev_query_pos) { + return false; + } + } else if (bases_[state.node_id()] == + (UInt8)agent.query()[state.query_pos()]) { + state.set_query_pos(state.query_pos() + 1); + return true; + } + state.set_node_id(state.node_id() + 1); + ++louds_pos; + } while (louds_[louds_pos]); + return false; +} + +bool LoudsTrie::predictive_find_child(Agent &agent) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + + State &state = agent.state(); + const std::size_t cache_id = get_cache_id(state.node_id(), + agent.query()[state.query_pos()]); + if (state.node_id() == cache_[cache_id].parent()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!prefix_match(agent, cache_[cache_id].link())) { + return false; + } + } else { + state.key_buf().push_back(cache_[cache_id].label()); + state.set_query_pos(state.query_pos() + 1); + } + state.set_node_id(cache_[cache_id].child()); + return true; + } + + std::size_t louds_pos = louds_.select0(state.node_id()) + 1; + if (!louds_[louds_pos]) { + return false; + } + state.set_node_id(louds_pos - state.node_id() - 1); + std::size_t link_id = MARISA_INVALID_LINK_ID; + do { + if (link_flags_[state.node_id()]) { + link_id = update_link_id(link_id, state.node_id()); + const std::size_t prev_query_pos = state.query_pos(); + if (prefix_match(agent, get_link(state.node_id(), link_id))) { + return true; + } else if (state.query_pos() != prev_query_pos) { + return false; + } + } else if (bases_[state.node_id()] == + (UInt8)agent.query()[state.query_pos()]) { + state.key_buf().push_back((char)bases_[state.node_id()]); + state.set_query_pos(state.query_pos() + 1); + return true; + } + state.set_node_id(state.node_id() + 1); + ++louds_pos; + } while (louds_[louds_pos]); + return false; +} + +void LoudsTrie::restore(Agent &agent, std::size_t link) const { + if (next_trie_.get() != NULL) { + next_trie_->restore_(agent, link); + } else { + tail_.restore(agent, link); + } +} + +bool LoudsTrie::match(Agent &agent, std::size_t link) const { + if (next_trie_.get() != NULL) { + return next_trie_->match_(agent, link); + } else { + return tail_.match(agent, link); + } +} + +bool LoudsTrie::prefix_match(Agent &agent, std::size_t link) const { + if (next_trie_.get() != NULL) { + return next_trie_->prefix_match_(agent, link); + } else { + return tail_.prefix_match(agent, link); + } +} + +void LoudsTrie::restore_(Agent &agent, std::size_t node_id) const { + MARISA_DEBUG_IF(node_id == 0, MARISA_RANGE_ERROR); + + State &state = agent.state(); + for ( ; ; ) { + const std::size_t cache_id = get_cache_id(node_id); + if (node_id == cache_[cache_id].child()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + restore(agent, cache_[cache_id].link()); + } else { + state.key_buf().push_back(cache_[cache_id].label()); + } + + node_id = cache_[cache_id].parent(); + if (node_id == 0) { + return; + } + continue; + } + + if (link_flags_[node_id]) { + restore(agent, get_link(node_id)); + } else { + state.key_buf().push_back((char)bases_[node_id]); + } + + if (node_id <= num_l1_nodes_) { + return; + } + node_id = louds_.select1(node_id) - node_id - 1; + } +} + +bool LoudsTrie::match_(Agent &agent, std::size_t node_id) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(node_id == 0, MARISA_RANGE_ERROR); + + State &state = agent.state(); + for ( ; ; ) { + const std::size_t cache_id = get_cache_id(node_id); + if (node_id == cache_[cache_id].child()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!match(agent, cache_[cache_id].link())) { + return false; + } + } else if (cache_[cache_id].label() == + agent.query()[state.query_pos()]) { + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + node_id = cache_[cache_id].parent(); + if (node_id == 0) { + return true; + } else if (state.query_pos() >= agent.query().length()) { + return false; + } + continue; + } + + if (link_flags_[node_id]) { + if (next_trie_.get() != NULL) { + if (!match(agent, get_link(node_id))) { + return false; + } + } else if (!tail_.match(agent, get_link(node_id))) { + return false; + } + } else if (bases_[node_id] == (UInt8)agent.query()[state.query_pos()]) { + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + if (node_id <= num_l1_nodes_) { + return true; + } else if (state.query_pos() >= agent.query().length()) { + return false; + } + node_id = louds_.select1(node_id) - node_id - 1; + } +} + +bool LoudsTrie::prefix_match_(Agent &agent, std::size_t node_id) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(node_id == 0, MARISA_RANGE_ERROR); + + State &state = agent.state(); + for ( ; ; ) { + const std::size_t cache_id = get_cache_id(node_id); + if (node_id == cache_[cache_id].child()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!prefix_match(agent, cache_[cache_id].link())) { + return false; + } + } else if (cache_[cache_id].label() == + agent.query()[state.query_pos()]) { + state.key_buf().push_back(cache_[cache_id].label()); + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + node_id = cache_[cache_id].parent(); + if (node_id == 0) { + return true; + } + } else { + if (link_flags_[node_id]) { + if (!prefix_match(agent, get_link(node_id))) { + return false; + } + } else if (bases_[node_id] == (UInt8)agent.query()[state.query_pos()]) { + state.key_buf().push_back((char)bases_[node_id]); + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + if (node_id <= num_l1_nodes_) { + return true; + } + node_id = louds_.select1(node_id) - node_id - 1; + } + + if (state.query_pos() >= agent.query().length()) { + restore_(agent, node_id); + return true; + } + } +} + +std::size_t LoudsTrie::get_cache_id(std::size_t node_id, char label) const { + return (node_id ^ (node_id << 5) ^ (UInt8)label) & cache_mask_; +} + +std::size_t LoudsTrie::get_cache_id(std::size_t node_id) const { + return node_id & cache_mask_; +} + +std::size_t LoudsTrie::get_link(std::size_t node_id) const { + return bases_[node_id] | (extras_[link_flags_.rank1(node_id)] * 256); +} + +std::size_t LoudsTrie::get_link(std::size_t node_id, + std::size_t link_id) const { + return bases_[node_id] | (extras_[link_id] * 256); +} + +std::size_t LoudsTrie::update_link_id(std::size_t link_id, + std::size_t node_id) const { + return (link_id == MARISA_INVALID_LINK_ID) ? + link_flags_.rank1(node_id) : (link_id + 1); +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/louds-trie.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/louds-trie.h new file mode 100644 index 00000000000..24ae013b25b --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/louds-trie.h @@ -0,0 +1,134 @@ +#ifndef MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_ +#define MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_ + +#include "marisa/keyset.h" +#include "marisa/agent.h" +#include "marisa/grimoire/vector.h" +#include "marisa/grimoire/trie/config.h" +#include "marisa/grimoire/trie/key.h" +#include "marisa/grimoire/trie/tail.h" +#include "marisa/grimoire/trie/cache.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class LoudsTrie { + public: + LoudsTrie(); + ~LoudsTrie(); + + void build(Keyset &keyset, int flags); + + void map(Mapper &mapper); + void read(Reader &reader); + void write(Writer &writer) const; + + bool lookup(Agent &agent) const; + void reverse_lookup(Agent &agent) const; + bool common_prefix_search(Agent &agent) const; + bool predictive_search(Agent &agent) const; + + std::size_t num_tries() const { + return config_.num_tries(); + } + std::size_t num_keys() const { + return size(); + } + std::size_t num_nodes() const { + return (louds_.size() / 2) - 1; + } + + CacheLevel cache_level() const { + return config_.cache_level(); + } + TailMode tail_mode() const { + return config_.tail_mode(); + } + NodeOrder node_order() const { + return config_.node_order(); + } + + bool empty() const { + return size() == 0; + } + std::size_t size() const { + return terminal_flags_.num_1s(); + } + std::size_t total_size() const; + std::size_t io_size() const; + + void clear(); + void swap(LoudsTrie &rhs); + + private: + BitVector louds_; + BitVector terminal_flags_; + BitVector link_flags_; + Vector<UInt8> bases_; + FlatVector extras_; + Tail tail_; + scoped_ptr<LoudsTrie> next_trie_; + Vector<Cache> cache_; + std::size_t cache_mask_; + std::size_t num_l1_nodes_; + Config config_; + Mapper mapper_; + + void build_(Keyset &keyset, const Config &config); + + template <typename T> + void build_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id); + template <typename T> + void build_current_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id); + template <typename T> + void build_next_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id); + template <typename T> + void build_terminals(const Vector<T> &keys, + Vector<UInt32> *terminals) const; + + void reserve_cache(const Config &config, std::size_t trie_id, + std::size_t num_keys); + template <typename T> + void cache(std::size_t parent, std::size_t child, + float weight, char label); + void fill_cache(); + + void map_(Mapper &mapper); + void read_(Reader &reader); + void write_(Writer &writer) const; + + inline bool find_child(Agent &agent) const; + inline bool predictive_find_child(Agent &agent) const; + + inline void restore(Agent &agent, std::size_t node_id) const; + inline bool match(Agent &agent, std::size_t node_id) const; + inline bool prefix_match(Agent &agent, std::size_t node_id) const; + + void restore_(Agent &agent, std::size_t node_id) const; + bool match_(Agent &agent, std::size_t node_id) const; + bool prefix_match_(Agent &agent, std::size_t node_id) const; + + inline std::size_t get_cache_id(std::size_t node_id, char label) const; + inline std::size_t get_cache_id(std::size_t node_id) const; + + inline std::size_t get_link(std::size_t node_id) const; + inline std::size_t get_link(std::size_t node_id, + std::size_t link_id) const; + + inline std::size_t update_link_id(std::size_t link_id, + std::size_t node_id) const; + + // Disallows copy and assignment. + LoudsTrie(const LoudsTrie &); + LoudsTrie &operator=(const LoudsTrie &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/range.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/range.h new file mode 100644 index 00000000000..6c78ddbd11e --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/range.h @@ -0,0 +1,115 @@ +#ifndef MARISA_GRIMOIRE_TRIE_RANGE_H_ +#define MARISA_GRIMOIRE_TRIE_RANGE_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Range { + public: + Range() : begin_(0), end_(0), key_pos_(0) {} + + void set_begin(std::size_t begin) { + MARISA_DEBUG_IF(begin > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + begin_ = static_cast<UInt32>(begin); + } + void set_end(std::size_t end) { + MARISA_DEBUG_IF(end > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + end_ = static_cast<UInt32>(end); + } + void set_key_pos(std::size_t key_pos) { + MARISA_DEBUG_IF(key_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_pos_ = static_cast<UInt32>(key_pos); + } + + std::size_t begin() const { + return begin_; + } + std::size_t end() const { + return end_; + } + std::size_t key_pos() const { + return key_pos_; + } + + private: + UInt32 begin_; + UInt32 end_; + UInt32 key_pos_; +}; + +inline Range make_range(std::size_t begin, std::size_t end, + std::size_t key_pos) { + Range range; + range.set_begin(begin); + range.set_end(end); + range.set_key_pos(key_pos); + return range; +} + +class WeightedRange { + public: + WeightedRange() : range_(), weight_(0.0F) {} + + void set_range(const Range &range) { + range_ = range; + } + void set_begin(std::size_t begin) { + range_.set_begin(begin); + } + void set_end(std::size_t end) { + range_.set_end(end); + } + void set_key_pos(std::size_t key_pos) { + range_.set_key_pos(key_pos); + } + void set_weight(float weight) { + weight_ = weight; + } + + const Range &range() const { + return range_; + } + std::size_t begin() const { + return range_.begin(); + } + std::size_t end() const { + return range_.end(); + } + std::size_t key_pos() const { + return range_.key_pos(); + } + float weight() const { + return weight_; + } + + private: + Range range_; + float weight_; +}; + +inline bool operator<(const WeightedRange &lhs, const WeightedRange &rhs) { + return lhs.weight() < rhs.weight(); +} + +inline bool operator>(const WeightedRange &lhs, const WeightedRange &rhs) { + return lhs.weight() > rhs.weight(); +} + +inline WeightedRange make_weighted_range(std::size_t begin, std::size_t end, + std::size_t key_pos, float weight) { + WeightedRange range; + range.set_begin(begin); + range.set_end(end); + range.set_key_pos(key_pos); + range.set_weight(weight); + return range; +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_RANGE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/state.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/state.h new file mode 100644 index 00000000000..df605a6a3e5 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/state.h @@ -0,0 +1,117 @@ +#ifndef MARISA_GRIMOIRE_TRIE_STATE_H_ +#define MARISA_GRIMOIRE_TRIE_STATE_H_ + +#include "marisa/grimoire/vector.h" +#include "marisa/grimoire/trie/history.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +// A search agent has its internal state and the status codes are defined +// below. +typedef enum StatusCode { + MARISA_READY_TO_ALL, + MARISA_READY_TO_COMMON_PREFIX_SEARCH, + MARISA_READY_TO_PREDICTIVE_SEARCH, + MARISA_END_OF_COMMON_PREFIX_SEARCH, + MARISA_END_OF_PREDICTIVE_SEARCH, +} StatusCode; + +class State { + public: + State() + : key_buf_(), history_(), node_id_(0), query_pos_(0), + history_pos_(0), status_code_(MARISA_READY_TO_ALL) {} + + void set_node_id(std::size_t node_id) { + MARISA_DEBUG_IF(node_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + node_id_ = (UInt32)node_id; + } + void set_query_pos(std::size_t query_pos) { + MARISA_DEBUG_IF(query_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + query_pos_ = (UInt32)query_pos; + } + void set_history_pos(std::size_t history_pos) { + MARISA_DEBUG_IF(history_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + history_pos_ = (UInt32)history_pos; + } + void set_status_code(StatusCode status_code) { + status_code_ = status_code; + } + + std::size_t node_id() const { + return node_id_; + } + std::size_t query_pos() const { + return query_pos_; + } + std::size_t history_pos() const { + return history_pos_; + } + StatusCode status_code() const { + return status_code_; + } + + const Vector<char> &key_buf() const { + return key_buf_; + } + const Vector<History> &history() const { + return history_; + } + + Vector<char> &key_buf() { + return key_buf_; + } + Vector<History> &history() { + return history_; + } + + void reset() { + status_code_ = MARISA_READY_TO_ALL; + } + + void lookup_init() { + node_id_ = 0; + query_pos_ = 0; + status_code_ = MARISA_READY_TO_ALL; + } + void reverse_lookup_init() { + key_buf_.resize(0); + key_buf_.reserve(32); + status_code_ = MARISA_READY_TO_ALL; + } + void common_prefix_search_init() { + node_id_ = 0; + query_pos_ = 0; + status_code_ = MARISA_READY_TO_COMMON_PREFIX_SEARCH; + } + void predictive_search_init() { + key_buf_.resize(0); + key_buf_.reserve(64); + history_.resize(0); + history_.reserve(4); + node_id_ = 0; + query_pos_ = 0; + history_pos_ = 0; + status_code_ = MARISA_READY_TO_PREDICTIVE_SEARCH; + } + + private: + Vector<char> key_buf_; + Vector<History> history_; + UInt32 node_id_; + UInt32 query_pos_; + UInt32 history_pos_; + StatusCode status_code_; + + // Disallows copy and assignment. + State(const State &); + State &operator=(const State &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_STATE_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/tail.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/tail.cc new file mode 100644 index 00000000000..bd9bd01d56e --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/tail.cc @@ -0,0 +1,218 @@ +#include "marisa/grimoire/algorithm.h" +#include "marisa/grimoire/trie/state.h" +#include "marisa/grimoire/trie/tail.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +Tail::Tail() : buf_(), end_flags_() {} + +void Tail::build(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode) { + MARISA_THROW_IF(offsets == NULL, MARISA_NULL_ERROR); + + switch (mode) { + case MARISA_TEXT_TAIL: { + for (std::size_t i = 0; i < entries.size(); ++i) { + const char * const ptr = entries[i].ptr(); + const std::size_t length = entries[i].length(); + for (std::size_t j = 0; j < length; ++j) { + if (ptr[j] == '\0') { + mode = MARISA_BINARY_TAIL; + break; + } + } + if (mode == MARISA_BINARY_TAIL) { + break; + } + } + break; + } + case MARISA_BINARY_TAIL: { + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined tail mode"); + } + } + + Tail temp; + temp.build_(entries, offsets, mode); + swap(temp); +} + +void Tail::map(Mapper &mapper) { + Tail temp; + temp.map_(mapper); + swap(temp); +} + +void Tail::read(Reader &reader) { + Tail temp; + temp.read_(reader); + swap(temp); +} + +void Tail::write(Writer &writer) const { + write_(writer); +} + +void Tail::restore(Agent &agent, std::size_t offset) const { + MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (end_flags_.empty()) { + for (const char *ptr = &buf_[offset]; *ptr != '\0'; ++ptr) { + state.key_buf().push_back(*ptr); + } + } else { + do { + state.key_buf().push_back(buf_[offset]); + } while (!end_flags_[offset++]); + } +} + +bool Tail::match(Agent &agent, std::size_t offset) const { + MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + + State &state = agent.state(); + if (end_flags_.empty()) { + const char * const ptr = &buf_[offset] - state.query_pos(); + do { + if (ptr[state.query_pos()] != agent.query()[state.query_pos()]) { + return false; + } + state.set_query_pos(state.query_pos() + 1); + if (ptr[state.query_pos()] == '\0') { + return true; + } + } while (state.query_pos() < agent.query().length()); + return false; + } else { + do { + if (buf_[offset] != agent.query()[state.query_pos()]) { + return false; + } + state.set_query_pos(state.query_pos() + 1); + if (end_flags_[offset++]) { + return true; + } + } while (state.query_pos() < agent.query().length()); + return false; + } +} + +bool Tail::prefix_match(Agent &agent, std::size_t offset) const { + MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (end_flags_.empty()) { + const char *ptr = &buf_[offset] - state.query_pos(); + do { + if (ptr[state.query_pos()] != agent.query()[state.query_pos()]) { + return false; + } + state.key_buf().push_back(ptr[state.query_pos()]); + state.set_query_pos(state.query_pos() + 1); + if (ptr[state.query_pos()] == '\0') { + return true; + } + } while (state.query_pos() < agent.query().length()); + ptr += state.query_pos(); + do { + state.key_buf().push_back(*ptr); + } while (*++ptr != '\0'); + return true; + } else { + do { + if (buf_[offset] != agent.query()[state.query_pos()]) { + return false; + } + state.key_buf().push_back(buf_[offset]); + state.set_query_pos(state.query_pos() + 1); + if (end_flags_[offset++]) { + return true; + } + } while (state.query_pos() < agent.query().length()); + do { + state.key_buf().push_back(buf_[offset]); + } while (!end_flags_[offset++]); + return true; + } +} + +void Tail::clear() { + Tail().swap(*this); +} + +void Tail::swap(Tail &rhs) { + buf_.swap(rhs.buf_); + end_flags_.swap(rhs.end_flags_); +} + +void Tail::build_(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode) { + for (std::size_t i = 0; i < entries.size(); ++i) { + entries[i].set_id(i); + } + Algorithm().sort(entries.begin(), entries.end()); + + Vector<UInt32> temp_offsets; + temp_offsets.resize(entries.size(), 0); + + const Entry dummy; + const Entry *last = &dummy; + for (std::size_t i = entries.size(); i > 0; --i) { + const Entry ¤t = entries[i - 1]; + MARISA_THROW_IF(current.length() == 0, MARISA_RANGE_ERROR); + std::size_t match = 0; + while ((match < current.length()) && (match < last->length()) && + ((*last)[match] == current[match])) { + ++match; + } + if ((match == current.length()) && (last->length() != 0)) { + temp_offsets[current.id()] = (UInt32)( + temp_offsets[last->id()] + (last->length() - match)); + } else { + temp_offsets[current.id()] = (UInt32)buf_.size(); + for (std::size_t j = 1; j <= current.length(); ++j) { + buf_.push_back(current[current.length() - j]); + } + if (mode == MARISA_TEXT_TAIL) { + buf_.push_back('\0'); + } else { + for (std::size_t j = 1; j < current.length(); ++j) { + end_flags_.push_back(false); + } + end_flags_.push_back(true); + } + MARISA_THROW_IF(buf_.size() > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + } + last = ¤t; + } + buf_.shrink(); + + offsets->swap(temp_offsets); +} + +void Tail::map_(Mapper &mapper) { + buf_.map(mapper); + end_flags_.map(mapper); +} + +void Tail::read_(Reader &reader) { + buf_.read(reader); + end_flags_.read(reader); +} + +void Tail::write_(Writer &writer) const { + buf_.write(writer); + end_flags_.write(writer); +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/tail.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/tail.h new file mode 100644 index 00000000000..dd24f3ef81f --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/trie/tail.h @@ -0,0 +1,72 @@ +#ifndef MARISA_GRIMOIRE_TRIE_TAIL_H_ +#define MARISA_GRIMOIRE_TRIE_TAIL_H_ + +#include "marisa/agent.h" +#include "marisa/grimoire/vector.h" +#include "marisa/grimoire/trie/entry.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Tail { + public: + Tail(); + + void build(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode); + + void map(Mapper &mapper); + void read(Reader &reader); + void write(Writer &writer) const; + + void restore(Agent &agent, std::size_t offset) const; + bool match(Agent &agent, std::size_t offset) const; + bool prefix_match(Agent &agent, std::size_t offset) const; + + const char &operator[](std::size_t offset) const { + MARISA_DEBUG_IF(offset >= buf_.size(), MARISA_BOUND_ERROR); + return buf_[offset]; + } + + TailMode mode() const { + return end_flags_.empty() ? MARISA_TEXT_TAIL : MARISA_BINARY_TAIL; + } + + bool empty() const { + return buf_.empty(); + } + std::size_t size() const { + return buf_.size(); + } + std::size_t total_size() const { + return buf_.total_size() + end_flags_.total_size(); + } + std::size_t io_size() const { + return buf_.io_size() + end_flags_.io_size(); + } + + void clear(); + void swap(Tail &rhs); + + private: + Vector<char> buf_; + BitVector end_flags_; + + void build_(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode); + + void map_(Mapper &mapper); + void read_(Reader &reader); + void write_(Writer &writer) const; + + // Disallows copy and assignment. + Tail(const Tail &); + Tail &operator=(const Tail &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_TAIL_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector.h new file mode 100644 index 00000000000..582160baf99 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector.h @@ -0,0 +1,18 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_H_ + +#include "marisa/grimoire/vector/vector.h" +#include "marisa/grimoire/vector/flat-vector.h" +#include "marisa/grimoire/vector/bit-vector.h" + +namespace marisa { +namespace grimoire { + +using vector::Vector; +typedef vector::FlatVector FlatVector; +typedef vector::BitVector BitVector; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc new file mode 100644 index 00000000000..60ded67fe63 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc @@ -0,0 +1,844 @@ +#include "marisa/grimoire/vector/pop-count.h" +#include "marisa/grimoire/vector/bit-vector.h" + +namespace marisa { +namespace grimoire { +namespace vector { +namespace { + +#ifdef MARISA_USE_BMI2 +std::size_t select_bit(std::size_t i, std::size_t bit_id, UInt64 unit) { + #ifdef _MSC_VER + unsigned long pos; + ::_BitScanForward64(&pos, _pdep_u64(1ULL << i, unit)); + return bit_id + pos; + #else // _MSC_VER + return bit_id + ::__builtin_ctzll(_pdep_u64(1ULL << i, unit)); + #endif // _MSC_VER +} +#else // MARISA_USE_BMI2 +const UInt8 SELECT_TABLE[8][256] = { + { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 + }, + { + 7, 7, 7, 1, 7, 2, 2, 1, 7, 3, 3, 1, 3, 2, 2, 1, + 7, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 6, 6, 1, 6, 2, 2, 1, 6, 3, 3, 1, 3, 2, 2, 1, + 6, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 6, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 7, 7, 1, 7, 2, 2, 1, 7, 3, 3, 1, 3, 2, 2, 1, + 7, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 6, 6, 1, 6, 2, 2, 1, 6, 3, 3, 1, 3, 2, 2, 1, + 6, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 6, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1 + }, + { + 7, 7, 7, 7, 7, 7, 7, 2, 7, 7, 7, 3, 7, 3, 3, 2, + 7, 7, 7, 4, 7, 4, 4, 2, 7, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 5, 7, 5, 5, 2, 7, 5, 5, 3, 5, 3, 3, 2, + 7, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 6, 7, 6, 6, 2, 7, 6, 6, 3, 6, 3, 3, 2, + 7, 6, 6, 4, 6, 4, 4, 2, 6, 4, 4, 3, 4, 3, 3, 2, + 7, 6, 6, 5, 6, 5, 5, 2, 6, 5, 5, 3, 5, 3, 3, 2, + 6, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 7, 7, 7, 7, 2, 7, 7, 7, 3, 7, 3, 3, 2, + 7, 7, 7, 4, 7, 4, 4, 2, 7, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 5, 7, 5, 5, 2, 7, 5, 5, 3, 5, 3, 3, 2, + 7, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 6, 7, 6, 6, 2, 7, 6, 6, 3, 6, 3, 3, 2, + 7, 6, 6, 4, 6, 4, 4, 2, 6, 4, 4, 3, 4, 3, 3, 2, + 7, 6, 6, 5, 6, 5, 5, 2, 6, 5, 5, 3, 5, 3, 3, 2, + 6, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 3, + 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 4, 7, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 3, + 7, 7, 7, 5, 7, 5, 5, 4, 7, 5, 5, 4, 5, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 3, + 7, 7, 7, 6, 7, 6, 6, 4, 7, 6, 6, 4, 6, 4, 4, 3, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 3, + 7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 3, + 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 4, 7, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 3, + 7, 7, 7, 5, 7, 5, 5, 4, 7, 5, 5, 4, 5, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 3, + 7, 7, 7, 6, 7, 6, 6, 4, 7, 6, 6, 4, 6, 4, 4, 3, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 3, + 7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 4, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 4, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 4 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 + } +}; + + #if MARISA_WORD_SIZE == 64 +const UInt64 MASK_01 = 0x0101010101010101ULL; + #if !defined(MARISA_X64) || !defined(MARISA_USE_SSSE3) +const UInt64 MASK_0F = 0x0F0F0F0F0F0F0F0FULL; +const UInt64 MASK_33 = 0x3333333333333333ULL; +const UInt64 MASK_55 = 0x5555555555555555ULL; + #endif // !defined(MARISA_X64) || !defined(MARISA_USE_SSSE3) + #if !defined(MARISA_X64) || !defined(MARISA_USE_POPCNT) +const UInt64 MASK_80 = 0x8080808080808080ULL; + #endif // !defined(MARISA_X64) || !defined(MARISA_USE_POPCNT) + +std::size_t select_bit(std::size_t i, std::size_t bit_id, UInt64 unit) { + UInt64 counts; + { + #if defined(MARISA_X64) && defined(MARISA_USE_SSSE3) + __m128i lower_nibbles = _mm_cvtsi64_si128( + static_cast<long long>(unit & 0x0F0F0F0F0F0F0F0FULL)); + __m128i upper_nibbles = _mm_cvtsi64_si128( + static_cast<long long>(unit & 0xF0F0F0F0F0F0F0F0ULL)); + upper_nibbles = _mm_srli_epi32(upper_nibbles, 4); + + __m128i lower_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + lower_counts = _mm_shuffle_epi8(lower_counts, lower_nibbles); + __m128i upper_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + upper_counts = _mm_shuffle_epi8(upper_counts, upper_nibbles); + + counts = static_cast<UInt64>(_mm_cvtsi128_si64( + _mm_add_epi8(lower_counts, upper_counts))); + #else // defined(MARISA_X64) && defined(MARISA_USE_SSSE3) + counts = unit - ((unit >> 1) & MASK_55); + counts = (counts & MASK_33) + ((counts >> 2) & MASK_33); + counts = (counts + (counts >> 4)) & MASK_0F; + #endif // defined(MARISA_X64) && defined(MARISA_USE_SSSE3) + counts *= MASK_01; + } + + #if defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + UInt8 skip; + { + __m128i x = _mm_cvtsi64_si128(static_cast<long long>((i + 1) * MASK_01)); + __m128i y = _mm_cvtsi64_si128(static_cast<long long>(counts)); + x = _mm_cmpgt_epi8(x, y); + skip = (UInt8)PopCount::count(static_cast<UInt64>(_mm_cvtsi128_si64(x))); + } + #else // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + const UInt64 x = (counts | MASK_80) - ((i + 1) * MASK_01); + #ifdef _MSC_VER + unsigned long skip; + ::_BitScanForward64(&skip, (x & MASK_80) >> 7); + #else // _MSC_VER + const int skip = ::__builtin_ctzll((x & MASK_80) >> 7); + #endif // _MSC_VER + #endif // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + + bit_id += static_cast<std::size_t>(skip); + unit >>= skip; + i -= ((counts << 8) >> skip) & 0xFF; + + return bit_id + SELECT_TABLE[i][unit & 0xFF]; +} + #else // MARISA_WORD_SIZE == 64 + #ifdef MARISA_USE_SSE2 +const UInt8 POPCNT_TABLE[256] = { + 0, 8, 8, 16, 8, 16, 16, 24, 8, 16, 16, 24, 16, 24, 24, 32, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 32, 40, 40, 48, 40, 48, 48, 56, 40, 48, 48, 56, 48, 56, 56, 64 +}; + +std::size_t select_bit(std::size_t i, std::size_t bit_id, + UInt32 unit_lo, UInt32 unit_hi) { + __m128i unit; + { + __m128i lower_dword = _mm_cvtsi32_si128(unit_lo); + __m128i upper_dword = _mm_cvtsi32_si128(unit_hi); + upper_dword = _mm_slli_si128(upper_dword, 4); + unit = _mm_or_si128(lower_dword, upper_dword); + } + + __m128i counts; + { + #ifdef MARISA_USE_SSSE3 + __m128i lower_nibbles = _mm_set1_epi8(0x0F); + lower_nibbles = _mm_and_si128(lower_nibbles, unit); + __m128i upper_nibbles = _mm_set1_epi8((UInt8)0xF0); + upper_nibbles = _mm_and_si128(upper_nibbles, unit); + upper_nibbles = _mm_srli_epi32(upper_nibbles, 4); + + __m128i lower_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + lower_counts = _mm_shuffle_epi8(lower_counts, lower_nibbles); + __m128i upper_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + upper_counts = _mm_shuffle_epi8(upper_counts, upper_nibbles); + + counts = _mm_add_epi8(lower_counts, upper_counts); + #else // MARISA_USE_SSSE3 + __m128i x = _mm_srli_epi32(unit, 1); + x = _mm_and_si128(x, _mm_set1_epi8(0x55)); + x = _mm_sub_epi8(unit, x); + + __m128i y = _mm_srli_epi32(x, 2); + y = _mm_and_si128(y, _mm_set1_epi8(0x33)); + x = _mm_and_si128(x, _mm_set1_epi8(0x33)); + x = _mm_add_epi8(x, y); + + y = _mm_srli_epi32(x, 4); + x = _mm_add_epi8(x, y); + counts = _mm_and_si128(x, _mm_set1_epi8(0x0F)); + #endif // MARISA_USE_SSSE3 + } + + __m128i accumulated_counts; + { + __m128i x = counts; + x = _mm_slli_si128(x, 1); + __m128i y = counts; + y = _mm_add_epi32(y, x); + + x = y; + y = _mm_slli_si128(y, 2); + x = _mm_add_epi32(x, y); + + y = x; + x = _mm_slli_si128(x, 4); + y = _mm_add_epi32(y, x); + + accumulated_counts = _mm_set_epi32(0x7F7F7F7FU, 0x7F7F7F7FU, 0, 0); + accumulated_counts = _mm_or_si128(accumulated_counts, y); + } + + UInt8 skip; + { + __m128i x = _mm_set1_epi8((UInt8)(i + 1)); + x = _mm_cmpgt_epi8(x, accumulated_counts); + skip = POPCNT_TABLE[_mm_movemask_epi8(x)]; + } + + UInt8 byte; + { + #ifdef _MSC_VER + __declspec(align(16)) UInt8 unit_bytes[16]; + __declspec(align(16)) UInt8 accumulated_counts_bytes[16]; + #else // _MSC_VER + UInt8 unit_bytes[16] __attribute__ ((aligned (16))); + UInt8 accumulated_counts_bytes[16] __attribute__ ((aligned (16))); + #endif // _MSC_VER + accumulated_counts = _mm_slli_si128(accumulated_counts, 1); + _mm_store_si128(reinterpret_cast<__m128i *>(unit_bytes), unit); + _mm_store_si128(reinterpret_cast<__m128i *>(accumulated_counts_bytes), + accumulated_counts); + + bit_id += skip; + byte = unit_bytes[skip / 8]; + i -= accumulated_counts_bytes[skip / 8]; + } + + return bit_id + SELECT_TABLE[i][byte]; +} + #endif // MARISA_USE_SSE2 + #endif // MARISA_WORD_SIZE == 64 +#endif // MARISA_USE_BMI2 + +} // namespace + +#if MARISA_WORD_SIZE == 64 + +std::size_t BitVector::rank1(std::size_t i) const { + MARISA_DEBUG_IF(ranks_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i > size_, MARISA_BOUND_ERROR); + + const RankIndex &rank = ranks_[i / 512]; + std::size_t offset = rank.abs(); + switch ((i / 64) % 8) { + case 1: { + offset += rank.rel1(); + break; + } + case 2: { + offset += rank.rel2(); + break; + } + case 3: { + offset += rank.rel3(); + break; + } + case 4: { + offset += rank.rel4(); + break; + } + case 5: { + offset += rank.rel5(); + break; + } + case 6: { + offset += rank.rel6(); + break; + } + case 7: { + offset += rank.rel7(); + break; + } + } + offset += PopCount::count(units_[i / 64] & ((1ULL << (i % 64)) - 1)); + return offset; +} + +std::size_t BitVector::select0(std::size_t i) const { + MARISA_DEBUG_IF(select0s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_0s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select0s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select0s_[select_id]; + } + std::size_t begin = select0s_[select_id] / 512; + std::size_t end = (select0s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ((begin + 1) * 512) - ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < (middle * 512) - ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= (rank_id * 512) - ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 8; + if (i < (256U - rank.rel4())) { + if (i < (128U - rank.rel2())) { + if (i >= (64U - rank.rel1())) { + unit_id += 1; + i -= 64 - rank.rel1(); + } + } else if (i < (192U - rank.rel3())) { + unit_id += 2; + i -= 128 - rank.rel2(); + } else { + unit_id += 3; + i -= 192 - rank.rel3(); + } + } else if (i < (384U - rank.rel6())) { + if (i < (320U - rank.rel5())) { + unit_id += 4; + i -= 256 - rank.rel4(); + } else { + unit_id += 5; + i -= 320 - rank.rel5(); + } + } else if (i < (448U - rank.rel7())) { + unit_id += 6; + i -= 384 - rank.rel6(); + } else { + unit_id += 7; + i -= 448 - rank.rel7(); + } + + return select_bit(i, unit_id * 64, ~units_[unit_id]); +} + +std::size_t BitVector::select1(std::size_t i) const { + MARISA_DEBUG_IF(select1s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_1s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select1s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select1s_[select_id]; + } + std::size_t begin = select1s_[select_id] / 512; + std::size_t end = (select1s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 8; + if (i < rank.rel4()) { + if (i < rank.rel2()) { + if (i >= rank.rel1()) { + unit_id += 1; + i -= rank.rel1(); + } + } else if (i < rank.rel3()) { + unit_id += 2; + i -= rank.rel2(); + } else { + unit_id += 3; + i -= rank.rel3(); + } + } else if (i < rank.rel6()) { + if (i < rank.rel5()) { + unit_id += 4; + i -= rank.rel4(); + } else { + unit_id += 5; + i -= rank.rel5(); + } + } else if (i < rank.rel7()) { + unit_id += 6; + i -= rank.rel6(); + } else { + unit_id += 7; + i -= rank.rel7(); + } + + return select_bit(i, unit_id * 64, units_[unit_id]); +} + +#else // MARISA_WORD_SIZE == 64 + +std::size_t BitVector::rank1(std::size_t i) const { + MARISA_DEBUG_IF(ranks_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i > size_, MARISA_BOUND_ERROR); + + const RankIndex &rank = ranks_[i / 512]; + std::size_t offset = rank.abs(); + switch ((i / 64) % 8) { + case 1: { + offset += rank.rel1(); + break; + } + case 2: { + offset += rank.rel2(); + break; + } + case 3: { + offset += rank.rel3(); + break; + } + case 4: { + offset += rank.rel4(); + break; + } + case 5: { + offset += rank.rel5(); + break; + } + case 6: { + offset += rank.rel6(); + break; + } + case 7: { + offset += rank.rel7(); + break; + } + } + if (((i / 32) & 1) == 1) { + offset += PopCount::count(units_[(i / 32) - 1]); + } + offset += PopCount::count(units_[i / 32] & ((1U << (i % 32)) - 1)); + return offset; +} + +std::size_t BitVector::select0(std::size_t i) const { + MARISA_DEBUG_IF(select0s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_0s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select0s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select0s_[select_id]; + } + std::size_t begin = select0s_[select_id] / 512; + std::size_t end = (select0s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ((begin + 1) * 512) - ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < (middle * 512) - ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= (rank_id * 512) - ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 16; + if (i < (256U - rank.rel4())) { + if (i < (128U - rank.rel2())) { + if (i >= (64U - rank.rel1())) { + unit_id += 2; + i -= 64 - rank.rel1(); + } + } else if (i < (192U - rank.rel3())) { + unit_id += 4; + i -= 128 - rank.rel2(); + } else { + unit_id += 6; + i -= 192 - rank.rel3(); + } + } else if (i < (384U - rank.rel6())) { + if (i < (320U - rank.rel5())) { + unit_id += 8; + i -= 256 - rank.rel4(); + } else { + unit_id += 10; + i -= 320 - rank.rel5(); + } + } else if (i < (448U - rank.rel7())) { + unit_id += 12; + i -= 384 - rank.rel6(); + } else { + unit_id += 14; + i -= 448 - rank.rel7(); + } + +#ifdef MARISA_USE_SSE2 + return select_bit(i, unit_id * 32, ~units_[unit_id], ~units_[unit_id + 1]); +#else // MARISA_USE_SSE2 + UInt32 unit = ~units_[unit_id]; + PopCount count(unit); + if (i >= count.lo32()) { + ++unit_id; + i -= count.lo32(); + unit = ~units_[unit_id]; + count = PopCount(unit); + } + + std::size_t bit_id = unit_id * 32; + if (i < count.lo16()) { + if (i >= count.lo8()) { + bit_id += 8; + unit >>= 8; + i -= count.lo8(); + } + } else if (i < count.lo24()) { + bit_id += 16; + unit >>= 16; + i -= count.lo16(); + } else { + bit_id += 24; + unit >>= 24; + i -= count.lo24(); + } + return bit_id + SELECT_TABLE[i][unit & 0xFF]; +#endif // MARISA_USE_SSE2 +} + +std::size_t BitVector::select1(std::size_t i) const { + MARISA_DEBUG_IF(select1s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_1s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select1s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select1s_[select_id]; + } + std::size_t begin = select1s_[select_id] / 512; + std::size_t end = (select1s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 16; + if (i < rank.rel4()) { + if (i < rank.rel2()) { + if (i >= rank.rel1()) { + unit_id += 2; + i -= rank.rel1(); + } + } else if (i < rank.rel3()) { + unit_id += 4; + i -= rank.rel2(); + } else { + unit_id += 6; + i -= rank.rel3(); + } + } else if (i < rank.rel6()) { + if (i < rank.rel5()) { + unit_id += 8; + i -= rank.rel4(); + } else { + unit_id += 10; + i -= rank.rel5(); + } + } else if (i < rank.rel7()) { + unit_id += 12; + i -= rank.rel6(); + } else { + unit_id += 14; + i -= rank.rel7(); + } + +#ifdef MARISA_USE_SSE2 + return select_bit(i, unit_id * 32, units_[unit_id], units_[unit_id + 1]); +#else // MARISA_USE_SSE2 + UInt32 unit = units_[unit_id]; + PopCount count(unit); + if (i >= count.lo32()) { + ++unit_id; + i -= count.lo32(); + unit = units_[unit_id]; + count = PopCount(unit); + } + + std::size_t bit_id = unit_id * 32; + if (i < count.lo16()) { + if (i >= count.lo8()) { + bit_id += 8; + unit >>= 8; + i -= count.lo8(); + } + } else if (i < count.lo24()) { + bit_id += 16; + unit >>= 16; + i -= count.lo16(); + } else { + bit_id += 24; + unit >>= 24; + i -= count.lo24(); + } + return bit_id + SELECT_TABLE[i][unit & 0xFF]; +#endif // MARISA_USE_SSE2 +} + +#endif // MARISA_WORD_SIZE == 64 + +void BitVector::build_index(const BitVector &bv, + bool enables_select0, bool enables_select1) { + ranks_.resize((bv.size() / 512) + (((bv.size() % 512) != 0) ? 1 : 0) + 1); + + std::size_t num_0s = 0; + std::size_t num_1s = 0; + + for (std::size_t i = 0; i < bv.size(); ++i) { + if ((i % 64) == 0) { + const std::size_t rank_id = i / 512; + switch ((i / 64) % 8) { + case 0: { + ranks_[rank_id].set_abs(num_1s); + break; + } + case 1: { + ranks_[rank_id].set_rel1(num_1s - ranks_[rank_id].abs()); + break; + } + case 2: { + ranks_[rank_id].set_rel2(num_1s - ranks_[rank_id].abs()); + break; + } + case 3: { + ranks_[rank_id].set_rel3(num_1s - ranks_[rank_id].abs()); + break; + } + case 4: { + ranks_[rank_id].set_rel4(num_1s - ranks_[rank_id].abs()); + break; + } + case 5: { + ranks_[rank_id].set_rel5(num_1s - ranks_[rank_id].abs()); + break; + } + case 6: { + ranks_[rank_id].set_rel6(num_1s - ranks_[rank_id].abs()); + break; + } + case 7: { + ranks_[rank_id].set_rel7(num_1s - ranks_[rank_id].abs()); + break; + } + } + } + + if (bv[i]) { + if (enables_select1 && ((num_1s % 512) == 0)) { + select1s_.push_back(static_cast<UInt32>(i)); + } + ++num_1s; + } else { + if (enables_select0 && ((num_0s % 512) == 0)) { + select0s_.push_back(static_cast<UInt32>(i)); + } + ++num_0s; + } + } + + if ((bv.size() % 512) != 0) { + const std::size_t rank_id = (bv.size() - 1) / 512; + switch (((bv.size() - 1) / 64) % 8) { + case 0: { + ranks_[rank_id].set_rel1(num_1s - ranks_[rank_id].abs()); + } // fall through + case 1: { + ranks_[rank_id].set_rel2(num_1s - ranks_[rank_id].abs()); + } // fall through + case 2: { + ranks_[rank_id].set_rel3(num_1s - ranks_[rank_id].abs()); + } // fall through + case 3: { + ranks_[rank_id].set_rel4(num_1s - ranks_[rank_id].abs()); + } // fall through + case 4: { + ranks_[rank_id].set_rel5(num_1s - ranks_[rank_id].abs()); + } // fall through + case 5: { + ranks_[rank_id].set_rel6(num_1s - ranks_[rank_id].abs()); + } // fall through + case 6: { + ranks_[rank_id].set_rel7(num_1s - ranks_[rank_id].abs()); + break; + } + } + } + + size_ = bv.size(); + num_1s_ = bv.num_1s(); + + ranks_.back().set_abs(num_1s); + if (enables_select0) { + select0s_.push_back(static_cast<UInt32>(bv.size())); + select0s_.shrink(); + } + if (enables_select1) { + select1s_.push_back(static_cast<UInt32>(bv.size())); + select1s_.shrink(); + } +} + +} // namespace vector +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/bit-vector.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/bit-vector.h new file mode 100644 index 00000000000..ea698f13311 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/bit-vector.h @@ -0,0 +1,179 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_BIT_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_BIT_VECTOR_H_ + +#include "marisa/grimoire/vector/rank-index.h" +#include "marisa/grimoire/vector/vector.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +class BitVector { + public: +#if MARISA_WORD_SIZE == 64 + typedef UInt64 Unit; +#else // MARISA_WORD_SIZE == 64 + typedef UInt32 Unit; +#endif // MARISA_WORD_SIZE == 64 + + BitVector() + : units_(), size_(0), num_1s_(0), ranks_(), select0s_(), select1s_() {} + + void build(bool enables_select0, bool enables_select1) { + BitVector temp; + temp.build_index(*this, enables_select0, enables_select1); + units_.shrink(); + temp.units_.swap(units_); + swap(temp); + } + + void map(Mapper &mapper) { + BitVector temp; + temp.map_(mapper); + swap(temp); + } + void read(Reader &reader) { + BitVector temp; + temp.read_(reader); + swap(temp); + } + void write(Writer &writer) const { + write_(writer); + } + + void disable_select0() { + select0s_.clear(); + } + void disable_select1() { + select1s_.clear(); + } + + void push_back(bool bit) { + MARISA_THROW_IF(size_ == MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + if (size_ == (MARISA_WORD_SIZE * units_.size())) { + units_.resize(units_.size() + (64 / MARISA_WORD_SIZE), 0); + } + if (bit) { + units_[size_ / MARISA_WORD_SIZE] |= + (Unit)1 << (size_ % MARISA_WORD_SIZE); + ++num_1s_; + } + ++size_; + } + + bool operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return (units_[i / MARISA_WORD_SIZE] + & ((Unit)1 << (i % MARISA_WORD_SIZE))) != 0; + } + + std::size_t rank0(std::size_t i) const { + MARISA_DEBUG_IF(ranks_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i > size_, MARISA_BOUND_ERROR); + return i - rank1(i); + } + std::size_t rank1(std::size_t i) const; + + std::size_t select0(std::size_t i) const; + std::size_t select1(std::size_t i) const; + + std::size_t num_0s() const { + return size_ - num_1s_; + } + std::size_t num_1s() const { + return num_1s_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + std::size_t total_size() const { + return units_.total_size() + ranks_.total_size() + + select0s_.total_size() + select1s_.total_size(); + } + std::size_t io_size() const { + return units_.io_size() + (sizeof(UInt32) * 2) + ranks_.io_size() + + select0s_.io_size() + select1s_.io_size(); + } + + void clear() { + BitVector().swap(*this); + } + void swap(BitVector &rhs) { + units_.swap(rhs.units_); + marisa::swap(size_, rhs.size_); + marisa::swap(num_1s_, rhs.num_1s_); + ranks_.swap(rhs.ranks_); + select0s_.swap(rhs.select0s_); + select1s_.swap(rhs.select1s_); + } + + private: + Vector<Unit> units_; + std::size_t size_; + std::size_t num_1s_; + Vector<RankIndex> ranks_; + Vector<UInt32> select0s_; + Vector<UInt32> select1s_; + + void build_index(const BitVector &bv, + bool enables_select0, bool enables_select1); + + void map_(Mapper &mapper) { + units_.map(mapper); + { + UInt32 temp_size; + mapper.map(&temp_size); + size_ = temp_size; + } + { + UInt32 temp_num_1s; + mapper.map(&temp_num_1s); + MARISA_THROW_IF(temp_num_1s > size_, MARISA_FORMAT_ERROR); + num_1s_ = temp_num_1s; + } + ranks_.map(mapper); + select0s_.map(mapper); + select1s_.map(mapper); + } + + void read_(Reader &reader) { + units_.read(reader); + { + UInt32 temp_size; + reader.read(&temp_size); + size_ = temp_size; + } + { + UInt32 temp_num_1s; + reader.read(&temp_num_1s); + MARISA_THROW_IF(temp_num_1s > size_, MARISA_FORMAT_ERROR); + num_1s_ = temp_num_1s; + } + ranks_.read(reader); + select0s_.read(reader); + select1s_.read(reader); + } + + void write_(Writer &writer) const { + units_.write(writer); + writer.write((UInt32)size_); + writer.write((UInt32)num_1s_); + ranks_.write(writer); + select0s_.write(writer); + select1s_.write(writer); + } + + // Disallows copy and assignment. + BitVector(const BitVector &); + BitVector &operator=(const BitVector &); +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_BIT_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/flat-vector.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/flat-vector.h new file mode 100644 index 00000000000..eeae719e27c --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/flat-vector.h @@ -0,0 +1,205 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_FLAT_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_FLAT_VECTOR_H_ + +#include "marisa/grimoire/vector/vector.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +class FlatVector { + public: +#if MARISA_WORD_SIZE == 64 + typedef UInt64 Unit; +#else // MARISA_WORD_SIZE == 64 + typedef UInt32 Unit; +#endif // MARISA_WORD_SIZE == 64 + + FlatVector() : units_(), value_size_(0), mask_(0), size_(0) {} + + void build(const Vector<UInt32> &values) { + FlatVector temp; + temp.build_(values); + swap(temp); + } + + void map(Mapper &mapper) { + FlatVector temp; + temp.map_(mapper); + swap(temp); + } + void read(Reader &reader) { + FlatVector temp; + temp.read_(reader); + swap(temp); + } + void write(Writer &writer) const { + write_(writer); + } + + UInt32 operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + + const std::size_t pos = i * value_size_; + const std::size_t unit_id = pos / MARISA_WORD_SIZE; + const std::size_t unit_offset = pos % MARISA_WORD_SIZE; + + if ((unit_offset + value_size_) <= MARISA_WORD_SIZE) { + return (UInt32)(units_[unit_id] >> unit_offset) & mask_; + } else { + return (UInt32)((units_[unit_id] >> unit_offset) + | (units_[unit_id + 1] << (MARISA_WORD_SIZE - unit_offset))) & mask_; + } + } + + std::size_t value_size() const { + return value_size_; + } + UInt32 mask() const { + return mask_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + std::size_t total_size() const { + return units_.total_size(); + } + std::size_t io_size() const { + return units_.io_size() + (sizeof(UInt32) * 2) + sizeof(UInt64); + } + + void clear() { + FlatVector().swap(*this); + } + void swap(FlatVector &rhs) { + units_.swap(rhs.units_); + marisa::swap(value_size_, rhs.value_size_); + marisa::swap(mask_, rhs.mask_); + marisa::swap(size_, rhs.size_); + } + + private: + Vector<Unit> units_; + std::size_t value_size_; + UInt32 mask_; + std::size_t size_; + + void build_(const Vector<UInt32> &values) { + UInt32 max_value = 0; + for (std::size_t i = 0; i < values.size(); ++i) { + if (values[i] > max_value) { + max_value = values[i]; + } + } + + std::size_t value_size = 0; + while (max_value != 0) { + ++value_size; + max_value >>= 1; + } + + std::size_t num_units = values.empty() ? 0 : (64 / MARISA_WORD_SIZE); + if (value_size != 0) { + num_units = (std::size_t)( + (((UInt64)value_size * values.size()) + (MARISA_WORD_SIZE - 1)) + / MARISA_WORD_SIZE); + num_units += num_units % (64 / MARISA_WORD_SIZE); + } + + units_.resize(num_units); + if (num_units > 0) { + units_.back() = 0; + } + + value_size_ = value_size; + if (value_size != 0) { + mask_ = MARISA_UINT32_MAX >> (32 - value_size); + } + size_ = values.size(); + + for (std::size_t i = 0; i < values.size(); ++i) { + set(i, values[i]); + } + } + + void map_(Mapper &mapper) { + units_.map(mapper); + { + UInt32 temp_value_size; + mapper.map(&temp_value_size); + MARISA_THROW_IF(temp_value_size > 32, MARISA_FORMAT_ERROR); + value_size_ = temp_value_size; + } + { + UInt32 temp_mask; + mapper.map(&temp_mask); + mask_ = temp_mask; + } + { + UInt64 temp_size; + mapper.map(&temp_size); + MARISA_THROW_IF(temp_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)temp_size; + } + } + + void read_(Reader &reader) { + units_.read(reader); + { + UInt32 temp_value_size; + reader.read(&temp_value_size); + MARISA_THROW_IF(temp_value_size > 32, MARISA_FORMAT_ERROR); + value_size_ = temp_value_size; + } + { + UInt32 temp_mask; + reader.read(&temp_mask); + mask_ = temp_mask; + } + { + UInt64 temp_size; + reader.read(&temp_size); + MARISA_THROW_IF(temp_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)temp_size; + } + } + + void write_(Writer &writer) const { + units_.write(writer); + writer.write((UInt32)value_size_); + writer.write((UInt32)mask_); + writer.write((UInt64)size_); + } + + void set(std::size_t i, UInt32 value) { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(value > mask_, MARISA_RANGE_ERROR); + + const std::size_t pos = i * value_size_; + const std::size_t unit_id = pos / MARISA_WORD_SIZE; + const std::size_t unit_offset = pos % MARISA_WORD_SIZE; + + units_[unit_id] &= ~((Unit)mask_ << unit_offset); + units_[unit_id] |= (Unit)(value & mask_) << unit_offset; + if ((unit_offset + value_size_) > MARISA_WORD_SIZE) { + units_[unit_id + 1] &= + ~((Unit)mask_ >> (MARISA_WORD_SIZE - unit_offset)); + units_[unit_id + 1] |= + (Unit)(value & mask_) >> (MARISA_WORD_SIZE - unit_offset); + } + } + + // Disallows copy and assignment. + FlatVector(const FlatVector &); + FlatVector &operator=(const FlatVector &); +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_FLAT_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/pop-count.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/pop-count.h new file mode 100644 index 00000000000..47f4b5db306 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/pop-count.h @@ -0,0 +1,110 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_POP_COUNT_H_ +#define MARISA_GRIMOIRE_VECTOR_POP_COUNT_H_ + +#include "marisa/grimoire/intrin.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +#if MARISA_WORD_SIZE == 64 + +class PopCount { + public: + explicit PopCount(UInt64 x) : value_() { + x = (x & 0x5555555555555555ULL) + ((x & 0xAAAAAAAAAAAAAAAAULL) >> 1); + x = (x & 0x3333333333333333ULL) + ((x & 0xCCCCCCCCCCCCCCCCULL) >> 2); + x = (x & 0x0F0F0F0F0F0F0F0FULL) + ((x & 0xF0F0F0F0F0F0F0F0ULL) >> 4); + x *= 0x0101010101010101ULL; + value_ = x; + } + + std::size_t lo8() const { + return (std::size_t)(value_ & 0xFFU); + } + std::size_t lo16() const { + return (std::size_t)((value_ >> 8) & 0xFFU); + } + std::size_t lo24() const { + return (std::size_t)((value_ >> 16) & 0xFFU); + } + std::size_t lo32() const { + return (std::size_t)((value_ >> 24) & 0xFFU); + } + std::size_t lo40() const { + return (std::size_t)((value_ >> 32) & 0xFFU); + } + std::size_t lo48() const { + return (std::size_t)((value_ >> 40) & 0xFFU); + } + std::size_t lo56() const { + return (std::size_t)((value_ >> 48) & 0xFFU); + } + std::size_t lo64() const { + return (std::size_t)((value_ >> 56) & 0xFFU); + } + + static std::size_t count(UInt64 x) { +#if defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + #ifdef _MSC_VER + return __popcnt64(x); + #else // _MSC_VER + return static_cast<std::size_t>(_mm_popcnt_u64(x)); + #endif // _MSC_VER +#else // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + return PopCount(x).lo64(); +#endif // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + } + + private: + UInt64 value_; +}; + +#else // MARISA_WORD_SIZE == 64 + +class PopCount { + public: + explicit PopCount(UInt32 x) : value_() { + x = (x & 0x55555555U) + ((x & 0xAAAAAAAAU) >> 1); + x = (x & 0x33333333U) + ((x & 0xCCCCCCCCU) >> 2); + x = (x & 0x0F0F0F0FU) + ((x & 0xF0F0F0F0U) >> 4); + x *= 0x01010101U; + value_ = x; + } + + std::size_t lo8() const { + return value_ & 0xFFU; + } + std::size_t lo16() const { + return (value_ >> 8) & 0xFFU; + } + std::size_t lo24() const { + return (value_ >> 16) & 0xFFU; + } + std::size_t lo32() const { + return (value_ >> 24) & 0xFFU; + } + + static std::size_t count(UInt32 x) { +#ifdef MARISA_USE_POPCNT + #ifdef _MSC_VER + return __popcnt(x); + #else // _MSC_VER + return _mm_popcnt_u32(x); + #endif // _MSC_VER +#else // MARISA_USE_POPCNT + return PopCount(x).lo32(); +#endif // MARISA_USE_POPCNT + } + + private: + UInt32 value_; +}; + +#endif // MARISA_WORD_SIZE == 64 + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_POP_COUNT_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/rank-index.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/rank-index.h new file mode 100644 index 00000000000..c1ce476be2f --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/rank-index.h @@ -0,0 +1,82 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_ +#define MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +class RankIndex { + public: + RankIndex() : abs_(0), rel_lo_(0), rel_hi_(0) {} + + void set_abs(std::size_t value) { + MARISA_DEBUG_IF(value > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + abs_ = (UInt32)value; + } + void set_rel1(std::size_t value) { + MARISA_DEBUG_IF(value > 64, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~0x7FU) | (value & 0x7FU)); + } + void set_rel2(std::size_t value) { + MARISA_DEBUG_IF(value > 128, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~(0xFFU << 7)) | ((value & 0xFFU) << 7)); + } + void set_rel3(std::size_t value) { + MARISA_DEBUG_IF(value > 192, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~(0xFFU << 15)) | ((value & 0xFFU) << 15)); + } + void set_rel4(std::size_t value) { + MARISA_DEBUG_IF(value > 256, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~(0x1FFU << 23)) | ((value & 0x1FFU) << 23)); + } + void set_rel5(std::size_t value) { + MARISA_DEBUG_IF(value > 320, MARISA_RANGE_ERROR); + rel_hi_ = (UInt32)((rel_hi_ & ~0x1FFU) | (value & 0x1FFU)); + } + void set_rel6(std::size_t value) { + MARISA_DEBUG_IF(value > 384, MARISA_RANGE_ERROR); + rel_hi_ = (UInt32)((rel_hi_ & ~(0x1FFU << 9)) | ((value & 0x1FFU) << 9)); + } + void set_rel7(std::size_t value) { + MARISA_DEBUG_IF(value > 448, MARISA_RANGE_ERROR); + rel_hi_ = (UInt32)((rel_hi_ & ~(0x1FFU << 18)) | ((value & 0x1FFU) << 18)); + } + + std::size_t abs() const { + return abs_; + } + std::size_t rel1() const { + return rel_lo_ & 0x7FU; + } + std::size_t rel2() const { + return (rel_lo_ >> 7) & 0xFFU; + } + std::size_t rel3() const { + return (rel_lo_ >> 15) & 0xFFU; + } + std::size_t rel4() const { + return (rel_lo_ >> 23) & 0x1FFU; + } + std::size_t rel5() const { + return rel_hi_ & 0x1FFU; + } + std::size_t rel6() const { + return (rel_hi_ >> 9) & 0x1FFU; + } + std::size_t rel7() const { + return (rel_hi_ >> 18) & 0x1FFU; + } + + private: + UInt32 abs_; + UInt32 rel_lo_; + UInt32 rel_hi_; +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/vector.h b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/vector.h new file mode 100644 index 00000000000..2bfccdbd173 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/grimoire/vector/vector.h @@ -0,0 +1,256 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_VECTOR_H_ + +#include <new> + +#include "marisa/grimoire/io.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +template <typename T> +class Vector { + public: + Vector() + : buf_(), objs_(NULL), const_objs_(NULL), + size_(0), capacity_(0), fixed_(false) {} + ~Vector() { + if (objs_ != NULL) { + for (std::size_t i = 0; i < size_; ++i) { + objs_[i].~T(); + } + } + } + + void map(Mapper &mapper) { + Vector temp; + temp.map_(mapper); + swap(temp); + } + + void read(Reader &reader) { + Vector temp; + temp.read_(reader); + swap(temp); + } + + void write(Writer &writer) const { + write_(writer); + } + + void push_back(const T &x) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == max_size(), MARISA_SIZE_ERROR); + reserve(size_ + 1); + new (&objs_[size_]) T(x); + ++size_; + } + + void pop_back() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + objs_[--size_].~T(); + } + + // resize() assumes that T's placement new does not throw an exception. + void resize(std::size_t size) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + reserve(size); + for (std::size_t i = size_; i < size; ++i) { + new (&objs_[i]) T; + } + for (std::size_t i = size; i < size_; ++i) { + objs_[i].~T(); + } + size_ = size; + } + + // resize() assumes that T's placement new does not throw an exception. + void resize(std::size_t size, const T &x) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + reserve(size); + for (std::size_t i = size_; i < size; ++i) { + new (&objs_[i]) T(x); + } + for (std::size_t i = size; i < size_; ++i) { + objs_[i].~T(); + } + size_ = size; + } + + void reserve(std::size_t capacity) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + if (capacity <= capacity_) { + return; + } + MARISA_DEBUG_IF(capacity > max_size(), MARISA_SIZE_ERROR); + std::size_t new_capacity = capacity; + if (capacity_ > (capacity / 2)) { + if (capacity_ > (max_size() / 2)) { + new_capacity = max_size(); + } else { + new_capacity = capacity_ * 2; + } + } + realloc(new_capacity); + } + + void shrink() { + MARISA_THROW_IF(fixed_, MARISA_STATE_ERROR); + if (size_ != capacity_) { + realloc(size_); + } + } + + void fix() { + MARISA_THROW_IF(fixed_, MARISA_STATE_ERROR); + fixed_ = true; + } + + const T *begin() const { + return const_objs_; + } + const T *end() const { + return const_objs_ + size_; + } + const T &operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return const_objs_[i]; + } + const T &front() const { + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return const_objs_[0]; + } + const T &back() const { + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return const_objs_[size_ - 1]; + } + + T *begin() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + return objs_; + } + T *end() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + return objs_ + size_; + } + T &operator[](std::size_t i) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return objs_[i]; + } + T &front() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return objs_[0]; + } + T &back() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return objs_[size_ - 1]; + } + + std::size_t size() const { + return size_; + } + std::size_t capacity() const { + return capacity_; + } + bool fixed() const { + return fixed_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t total_size() const { + return sizeof(T) * size_; + } + std::size_t io_size() const { + return sizeof(UInt64) + ((total_size() + 7) & ~(std::size_t)0x07); + } + + void clear() { + Vector().swap(*this); + } + void swap(Vector &rhs) { + buf_.swap(rhs.buf_); + marisa::swap(objs_, rhs.objs_); + marisa::swap(const_objs_, rhs.const_objs_); + marisa::swap(size_, rhs.size_); + marisa::swap(capacity_, rhs.capacity_); + marisa::swap(fixed_, rhs.fixed_); + } + + static std::size_t max_size() { + return MARISA_SIZE_MAX / sizeof(T); + } + + private: + scoped_array<char> buf_; + T *objs_; + const T *const_objs_; + std::size_t size_; + std::size_t capacity_; + bool fixed_; + + void map_(Mapper &mapper) { + UInt64 total_size; + mapper.map(&total_size); + MARISA_THROW_IF(total_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF((total_size % sizeof(T)) != 0, MARISA_FORMAT_ERROR); + const std::size_t size = (std::size_t)(total_size / sizeof(T)); + mapper.map(&const_objs_, size); + mapper.seek((std::size_t)((8 - (total_size % 8)) % 8)); + size_ = size; + fix(); + } + void read_(Reader &reader) { + UInt64 total_size; + reader.read(&total_size); + MARISA_THROW_IF(total_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF((total_size % sizeof(T)) != 0, MARISA_FORMAT_ERROR); + const std::size_t size = (std::size_t)(total_size / sizeof(T)); + resize(size); + reader.read(objs_, size); + reader.seek((std::size_t)((8 - (total_size % 8)) % 8)); + } + void write_(Writer &writer) const { + writer.write((UInt64)total_size()); + writer.write(const_objs_, size_); + writer.seek((8 - (total_size() % 8)) % 8); + } + + // realloc() assumes that T's placement new does not throw an exception. + void realloc(std::size_t new_capacity) { + MARISA_DEBUG_IF(new_capacity > max_size(), MARISA_SIZE_ERROR); + + scoped_array<char> new_buf( + new (std::nothrow) char[sizeof(T) * new_capacity]); + MARISA_DEBUG_IF(new_buf.get() == NULL, MARISA_MEMORY_ERROR); + T *new_objs = reinterpret_cast<T *>(new_buf.get()); + + for (std::size_t i = 0; i < size_; ++i) { + new (&new_objs[i]) T(objs_[i]); + } + for (std::size_t i = 0; i < size_; ++i) { + objs_[i].~T(); + } + + buf_.swap(new_buf); + objs_ = new_objs; + const_objs_ = new_objs; + capacity_ = new_capacity; + } + + // Disallows copy and assignment. + Vector(const Vector &); + Vector &operator=(const Vector &); +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/keyset.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/keyset.cc new file mode 100644 index 00000000000..41354f7144d --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/keyset.cc @@ -0,0 +1,181 @@ +#include <new> + +#include "marisa/keyset.h" + +namespace marisa { + +Keyset::Keyset() + : base_blocks_(), base_blocks_size_(0), base_blocks_capacity_(0), + extra_blocks_(), extra_blocks_size_(0), extra_blocks_capacity_(0), + key_blocks_(), key_blocks_size_(0), key_blocks_capacity_(0), + ptr_(NULL), avail_(0), size_(0), total_length_(0) {} + +void Keyset::push_back(const Key &key) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + + char * const key_ptr = reserve(key.length()); + for (std::size_t i = 0; i < key.length(); ++i) { + key_ptr[i] = key[i]; + } + + Key &new_key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE]; + new_key.set_str(key_ptr, key.length()); + new_key.set_id(key.id()); + ++size_; + total_length_ += new_key.length(); +} + +void Keyset::push_back(const Key &key, char end_marker) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + + if ((size_ / KEY_BLOCK_SIZE) == key_blocks_size_) { + append_key_block(); + } + + char * const key_ptr = reserve(key.length() + 1); + for (std::size_t i = 0; i < key.length(); ++i) { + key_ptr[i] = key[i]; + } + key_ptr[key.length()] = end_marker; + + Key &new_key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE]; + new_key.set_str(key_ptr, key.length()); + new_key.set_id(key.id()); + ++size_; + total_length_ += new_key.length(); +} + +void Keyset::push_back(const char *str) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF(str == NULL, MARISA_NULL_ERROR); + + std::size_t length = 0; + while (str[length] != '\0') { + ++length; + } + push_back(str, length); +} + +void Keyset::push_back(const char *ptr, std::size_t length, float weight) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + + char * const key_ptr = reserve(length); + for (std::size_t i = 0; i < length; ++i) { + key_ptr[i] = ptr[i]; + } + + Key &key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE]; + key.set_str(key_ptr, length); + key.set_weight(weight); + ++size_; + total_length_ += length; +} + +void Keyset::reset() { + base_blocks_size_ = 0; + extra_blocks_size_ = 0; + ptr_ = NULL; + avail_ = 0; + size_ = 0; + total_length_ = 0; +} + +void Keyset::clear() { + Keyset().swap(*this); +} + +void Keyset::swap(Keyset &rhs) { + base_blocks_.swap(rhs.base_blocks_); + marisa::swap(base_blocks_size_, rhs.base_blocks_size_); + marisa::swap(base_blocks_capacity_, rhs.base_blocks_capacity_); + extra_blocks_.swap(rhs.extra_blocks_); + marisa::swap(extra_blocks_size_, rhs.extra_blocks_size_); + marisa::swap(extra_blocks_capacity_, rhs.extra_blocks_capacity_); + key_blocks_.swap(rhs.key_blocks_); + marisa::swap(key_blocks_size_, rhs.key_blocks_size_); + marisa::swap(key_blocks_capacity_, rhs.key_blocks_capacity_); + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(avail_, rhs.avail_); + marisa::swap(size_, rhs.size_); + marisa::swap(total_length_, rhs.total_length_); +} + +char *Keyset::reserve(std::size_t size) { + if ((size_ / KEY_BLOCK_SIZE) == key_blocks_size_) { + append_key_block(); + } + + if (size > EXTRA_BLOCK_SIZE) { + append_extra_block(size); + return extra_blocks_[extra_blocks_size_ - 1].get(); + } else { + if (size > avail_) { + append_base_block(); + } + ptr_ += size; + avail_ -= size; + return ptr_ - size; + } +} + +void Keyset::append_base_block() { + if (base_blocks_size_ == base_blocks_capacity_) { + const std::size_t new_capacity = + (base_blocks_size_ != 0) ? (base_blocks_size_ * 2) : 1; + scoped_array<scoped_array<char> > new_blocks( + new (std::nothrow) scoped_array<char>[new_capacity]); + MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR); + for (std::size_t i = 0; i < base_blocks_size_; ++i) { + base_blocks_[i].swap(new_blocks[i]); + } + base_blocks_.swap(new_blocks); + base_blocks_capacity_ = new_capacity; + } + if (base_blocks_[base_blocks_size_].get() == NULL) { + scoped_array<char> new_block(new (std::nothrow) char[BASE_BLOCK_SIZE]); + MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR); + base_blocks_[base_blocks_size_].swap(new_block); + } + ptr_ = base_blocks_[base_blocks_size_++].get(); + avail_ = BASE_BLOCK_SIZE; +} + +void Keyset::append_extra_block(std::size_t size) { + if (extra_blocks_size_ == extra_blocks_capacity_) { + const std::size_t new_capacity = + (extra_blocks_size_ != 0) ? (extra_blocks_size_ * 2) : 1; + scoped_array<scoped_array<char> > new_blocks( + new (std::nothrow) scoped_array<char>[new_capacity]); + MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR); + for (std::size_t i = 0; i < extra_blocks_size_; ++i) { + extra_blocks_[i].swap(new_blocks[i]); + } + extra_blocks_.swap(new_blocks); + extra_blocks_capacity_ = new_capacity; + } + scoped_array<char> new_block(new (std::nothrow) char[size]); + MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR); + extra_blocks_[extra_blocks_size_++].swap(new_block); +} + +void Keyset::append_key_block() { + if (key_blocks_size_ == key_blocks_capacity_) { + const std::size_t new_capacity = + (key_blocks_size_ != 0) ? (key_blocks_size_ * 2) : 1; + scoped_array<scoped_array<Key> > new_blocks( + new (std::nothrow) scoped_array<Key>[new_capacity]); + MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR); + for (std::size_t i = 0; i < key_blocks_size_; ++i) { + key_blocks_[i].swap(new_blocks[i]); + } + key_blocks_.swap(new_blocks); + key_blocks_capacity_ = new_capacity; + } + scoped_array<Key> new_block(new (std::nothrow) Key[KEY_BLOCK_SIZE]); + MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR); + key_blocks_[key_blocks_size_++].swap(new_block); +} + +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/trie.cc b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/trie.cc new file mode 100644 index 00000000000..68050011cc1 --- /dev/null +++ b/contrib/python/marisa-trie/py2/marisa-trie/lib/marisa/trie.cc @@ -0,0 +1,249 @@ +#include "marisa/stdio.h" +#include "marisa/iostream.h" +#include "marisa/trie.h" +#include "marisa/grimoire/trie.h" + +namespace marisa { + +Trie::Trie() : trie_() {} + +Trie::~Trie() {} + +void Trie::build(Keyset &keyset, int config_flags) { + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + temp->build(keyset, config_flags); + trie_.swap(temp); +} + +void Trie::mmap(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Mapper mapper; + mapper.open(filename); + temp->map(mapper); + trie_.swap(temp); +} + +void Trie::map(const void *ptr, std::size_t size) { + MARISA_THROW_IF((ptr == NULL) && (size != 0), MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Mapper mapper; + mapper.open(ptr, size); + temp->map(mapper); + trie_.swap(temp); +} + +void Trie::load(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(filename); + temp->read(reader); + trie_.swap(temp); +} + +void Trie::read(int fd) { + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(fd); + temp->read(reader); + trie_.swap(temp); +} + +void Trie::save(const char *filename) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + grimoire::Writer writer; + writer.open(filename); + trie_->write(writer); +} + +void Trie::write(int fd) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + grimoire::Writer writer; + writer.open(fd); + trie_->write(writer); +} + +bool Trie::lookup(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + return trie_->lookup(agent); +} + +void Trie::reverse_lookup(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + trie_->reverse_lookup(agent); +} + +bool Trie::common_prefix_search(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + return trie_->common_prefix_search(agent); +} + +bool Trie::predictive_search(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + return trie_->predictive_search(agent); +} + +std::size_t Trie::num_tries() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->num_tries(); +} + +std::size_t Trie::num_keys() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->num_keys(); +} + +std::size_t Trie::num_nodes() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->num_nodes(); +} + +TailMode Trie::tail_mode() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->tail_mode(); +} + +NodeOrder Trie::node_order() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->node_order(); +} + +bool Trie::empty() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->empty(); +} + +std::size_t Trie::size() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->size(); +} + +std::size_t Trie::total_size() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->total_size(); +} + +std::size_t Trie::io_size() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->io_size(); +} + +void Trie::clear() { + Trie().swap(*this); +} + +void Trie::swap(Trie &rhs) { + trie_.swap(rhs.trie_); +} + +} // namespace marisa + +#include <iostream> + +namespace marisa { + +class TrieIO { + public: + static void fread(std::FILE *file, Trie *trie) { + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp( + new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(file); + temp->read(reader); + trie->trie_.swap(temp); + } + static void fwrite(std::FILE *file, const Trie &trie) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + MARISA_THROW_IF(trie.trie_.get() == NULL, MARISA_STATE_ERROR); + grimoire::Writer writer; + writer.open(file); + trie.trie_->write(writer); + } + + static std::istream &read(std::istream &stream, Trie *trie) { + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp( + new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(stream); + temp->read(reader); + trie->trie_.swap(temp); + return stream; + } + static std::ostream &write(std::ostream &stream, const Trie &trie) { + MARISA_THROW_IF(trie.trie_.get() == NULL, MARISA_STATE_ERROR); + grimoire::Writer writer; + writer.open(stream); + trie.trie_->write(writer); + return stream; + } +}; + +void fread(std::FILE *file, Trie *trie) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + TrieIO::fread(file, trie); +} + +void fwrite(std::FILE *file, const Trie &trie) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + TrieIO::fwrite(file, trie); +} + +std::istream &read(std::istream &stream, Trie *trie) { + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + return TrieIO::read(stream, trie); +} + +std::ostream &write(std::ostream &stream, const Trie &trie) { + return TrieIO::write(stream, trie); +} + +std::istream &operator>>(std::istream &stream, Trie &trie) { + return read(stream, &trie); +} + +std::ostream &operator<<(std::ostream &stream, const Trie &trie) { + return write(stream, trie); +} + +} // namespace marisa diff --git a/contrib/python/marisa-trie/py2/patches/01-throw.patch b/contrib/python/marisa-trie/py2/patches/01-throw.patch new file mode 100644 index 00000000000..790d2c5606a --- /dev/null +++ b/contrib/python/marisa-trie/py2/patches/01-throw.patch @@ -0,0 +1,20 @@ +--- contrib/python/marisa-trie/py2/marisa-trie/include/marisa/exception.h (index) ++++ contrib/python/marisa-trie/py2/marisa-trie/include/marisa/exception.h (working tree) +@@ -19,7 +21,7 @@ class Exception : public std::exception { + Exception(const Exception &ex) + : std::exception(), filename_(ex.filename_), line_(ex.line_), + error_code_(ex.error_code_), error_message_(ex.error_message_) {} +- virtual ~Exception() throw() {} ++ virtual ~Exception() {} + + Exception &operator=(const Exception &rhs) { + filename_ = rhs.filename_; +@@ -42,7 +44,7 @@ class Exception : public std::exception { + return error_message_; + } + +- virtual const char *what() const throw() { ++ virtual const char *what() const noexcept { + return error_message_; + } + diff --git a/contrib/python/marisa-trie/py2/patches/03-fix-ya.make.patch b/contrib/python/marisa-trie/py2/patches/03-fix-ya.make.patch new file mode 100644 index 00000000000..ed06b9aa880 --- /dev/null +++ b/contrib/python/marisa-trie/py2/patches/03-fix-ya.make.patch @@ -0,0 +1,11 @@ +--- contrib/python/marisa-trie/py2/ya.make (index) ++++ contrib/python/marisa-trie/py2/ya.make (working tree) +@@ -32,7 +32,7 @@ SRCS( + PY_SRCS( + TOP_LEVEL + CYTHON_CPP +- src/marisa_trie.pyx ++ src/marisa_trie.pyx=marisa_trie + ) + + RESOURCE_FILES( diff --git a/contrib/python/marisa-trie/py2/src/agent.pxd b/contrib/python/marisa-trie/py2/src/agent.pxd new file mode 100644 index 00000000000..bf019673c2c --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/agent.pxd @@ -0,0 +1,22 @@ +cimport query, key + +cdef extern from "<marisa/agent.h>" namespace "marisa" nogil: + cdef cppclass Agent: + Agent() except + + + query.Query &query() + key.Key &key() + + void set_query(char *str) + void set_query(char *ptr, int length) + void set_query(int key_id) + + void set_key(char *str) + void set_key(char *ptr, int length) + void set_key(int id) + + void clear() + + void init_state() + + void swap(Agent &rhs) diff --git a/contrib/python/marisa-trie/py2/src/base.pxd b/contrib/python/marisa-trie/py2/src/base.pxd new file mode 100644 index 00000000000..c434e82122c --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/base.pxd @@ -0,0 +1,63 @@ +cdef extern from "<marisa/base.h>": + + # A dictionary consists of 3 tries in default. Usually more tries make a + # dictionary space-efficient but time-inefficient. + ctypedef enum marisa_num_tries: + MARISA_MIN_NUM_TRIES + MARISA_MAX_NUM_TRIES + MARISA_DEFAULT_NUM_TRIES + + + # This library uses a cache technique to accelerate search functions. The + # following enumerated type marisa_cache_level gives a list of available cache + # size options. A larger cache enables faster search but takes a more space. + ctypedef enum marisa_cache_level: + MARISA_HUGE_CACHE + MARISA_LARGE_CACHE + MARISA_NORMAL_CACHE + MARISA_SMALL_CACHE + MARISA_TINY_CACHE + MARISA_DEFAULT_CACHE + + # This library provides 2 kinds of TAIL implementations. + ctypedef enum marisa_tail_mode: + # MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is + # available if and only if the last labels do not contain a NULL character. + # If MARISA_TEXT_TAIL is specified and a NULL character exists in the last + # labels, the setting is automatically switched to MARISA_BINARY_TAIL. + MARISA_TEXT_TAIL + + # MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses + # a bit vector to detect the end of a sequence, instead of NULL characters. + # So, MARISA_BINARY_TAIL requires a larger space if the average length of + # labels is greater than 8. + MARISA_BINARY_TAIL + + MARISA_DEFAULT_TAIL + + # The arrangement of nodes affects the time cost of matching and the order of + # predictive search. + ctypedef enum marisa_node_order: + # MARISA_LABEL_ORDER arranges nodes in ascending label order. + # MARISA_LABEL_ORDER is useful if an application needs to predict keys in + # label order. + MARISA_LABEL_ORDER + + # MARISA_WEIGHT_ORDER arranges nodes in descending weight order. + # MARISA_WEIGHT_ORDER is generally a better choice because it enables faster + # matching. + MARISA_WEIGHT_ORDER + MARISA_DEFAULT_ORDER + + ctypedef enum marisa_config_mask: + MARISA_NUM_TRIES_MASK + MARISA_CACHE_LEVEL_MASK + MARISA_TAIL_MODE_MASK + MARISA_NODE_ORDER_MASK + MARISA_CONFIG_MASK + + +cdef extern from "<marisa/base.h>" namespace "marisa": + ctypedef marisa_cache_level CacheLevel + ctypedef marisa_tail_mode TailMode + ctypedef marisa_node_order NodeOrder diff --git a/contrib/python/marisa-trie/py2/src/iostream.pxd b/contrib/python/marisa-trie/py2/src/iostream.pxd new file mode 100644 index 00000000000..435ee85bb0b --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/iostream.pxd @@ -0,0 +1,7 @@ +from std_iostream cimport istream, ostream +from trie cimport Trie + +cdef extern from "<marisa/iostream.h>" namespace "marisa" nogil: + + istream &read(istream &stream, Trie *trie) + ostream &write(ostream &stream, Trie &trie) diff --git a/contrib/python/marisa-trie/py2/src/key.pxd b/contrib/python/marisa-trie/py2/src/key.pxd new file mode 100644 index 00000000000..d99dee5e04a --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/key.pxd @@ -0,0 +1,22 @@ +cdef extern from "<marisa/key.h>" namespace "marisa" nogil: + + cdef cppclass Key: + Key() + Key(Key &query) + + #Key &operator=(Key &query) + + char operator[](int i) + + void set_str(char *str) + void set_str(char *ptr, int length) + void set_id(int id) + void set_weight(float weight) + + char *ptr() + int length() + int id() + float weight() + + void clear() + void swap(Key &rhs) diff --git a/contrib/python/marisa-trie/py2/src/keyset.pxd b/contrib/python/marisa-trie/py2/src/keyset.pxd new file mode 100644 index 00000000000..1fb99a40c55 --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/keyset.pxd @@ -0,0 +1,30 @@ +cimport key + +cdef extern from "<marisa/keyset.h>" namespace "marisa" nogil: + cdef cppclass Keyset: + +# cdef enum constants: +# BASE_BLOCK_SIZE = 4096 +# EXTRA_BLOCK_SIZE = 1024 +# KEY_BLOCK_SIZE = 256 + + Keyset() + + void push_back(key.Key &key) + void push_back(key.Key &key, char end_marker) + + void push_back(char *str) + void push_back(char *ptr, int length) + void push_back(char *ptr, int length, float weight) + + key.Key &operator[](int i) + + int num_keys() + bint empty() + + int size() + int total_length() + + void reset() + void clear() + void swap(Keyset &rhs) diff --git a/contrib/python/marisa-trie/py2/src/marisa_trie.pyx b/contrib/python/marisa-trie/py2/src/marisa_trie.pyx new file mode 100644 index 00000000000..4dfb38b95b1 --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/marisa_trie.pyx @@ -0,0 +1,774 @@ +# cython: profile=False, embedsignature=True + +from __future__ import unicode_literals + +from std_iostream cimport stringstream, istream, ostream +from libc.string cimport strncmp +cimport keyset +cimport key +cimport agent +cimport trie +cimport iostream +cimport base + +import itertools +import struct +import warnings + +try: + from itertools import izip +except ImportError: + izip = zip + + +DEFAULT_CACHE = base.MARISA_DEFAULT_CACHE +HUGE_CACHE = base.MARISA_HUGE_CACHE +LARGE_CACHE = base.MARISA_LARGE_CACHE +NORMAL_CACHE = base.MARISA_NORMAL_CACHE +SMALL_CACHE = base.MARISA_SMALL_CACHE +TINY_CACHE = base.MARISA_TINY_CACHE + +MIN_NUM_TRIES = base.MARISA_MIN_NUM_TRIES +MAX_NUM_TRIES = base.MARISA_MAX_NUM_TRIES +DEFAULT_NUM_TRIES = base.MARISA_DEFAULT_NUM_TRIES + +# MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is +# available if and only if the last labels do not contain a NULL character. +# If MARISA_TEXT_TAIL is specified and a NULL character exists in the last +# labels, the setting is automatically switched to MARISA_BINARY_TAIL. +TEXT_TAIL = base.MARISA_TEXT_TAIL + +# MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses +# a bit vector to detect the end of a sequence, instead of NULL characters. +# So, MARISA_BINARY_TAIL requires a larger space if the average length of +# labels is greater than 8. +BINARY_TAIL = base.MARISA_BINARY_TAIL +DEFAULT_TAIL = base.MARISA_DEFAULT_TAIL + + +# MARISA_LABEL_ORDER arranges nodes in ascending label order. +# MARISA_LABEL_ORDER is useful if an application needs to predict keys in +# label order. +LABEL_ORDER = base.MARISA_LABEL_ORDER + +# MARISA_WEIGHT_ORDER arranges nodes in descending weight order. +# MARISA_WEIGHT_ORDER is generally a better choice because it enables faster +# matching. +WEIGHT_ORDER = base.MARISA_WEIGHT_ORDER +DEFAULT_ORDER = base.MARISA_DEFAULT_ORDER + + +cdef class _Trie: + cdef trie.Trie* _trie + + cdef bytes _encode_key(self, key): + return key + + cdef _get_key(self, agent.Agent& ag): + return ag.key().ptr()[:ag.key().length()] + + def __init__(self, arg=None, num_tries=DEFAULT_NUM_TRIES, binary=False, + cache_size=DEFAULT_CACHE, order=DEFAULT_ORDER, weights=None): + """ + ``arg`` can be one of the following: + + * an iterable with bytes keys; + * None (if you're going to load a trie later). + + Pass a ``weights`` iterable with expected lookup frequencies + to optimize lookup and prefix search speed. + """ + + if self._trie: + return + self._trie = new trie.Trie() + + byte_keys = (self._encode_key(key) for key in (arg or [])) + + self._build( + byte_keys, + weights, + num_tries=num_tries, + binary=binary, + cache_size=cache_size, + order=order + ) + + def __dealloc__(self): + if self._trie: + del self._trie + + def _config_flags(self, num_tries=DEFAULT_NUM_TRIES, binary=False, + cache_size=DEFAULT_CACHE, order=DEFAULT_ORDER): + if not MIN_NUM_TRIES <= num_tries <= MAX_NUM_TRIES: + raise ValueError( + "num_tries (which is %d) must be between between %d and %d" % + (num_tries, MIN_NUM_TRIES, MAX_NUM_TRIES)) + + binary_flag = BINARY_TAIL if binary else TEXT_TAIL + return num_tries | binary_flag | cache_size | order + + def _build(self, byte_keys, weights=None, **options): + if weights is None: + weights = itertools.repeat(1.0) + + cdef char* data + cdef float weight + cdef keyset.Keyset *ks = new keyset.Keyset() + + try: + for key, weight in izip(byte_keys, weights): + ks.push_back(<char *>key, len(key), weight) + self._trie.build(ks[0], self._config_flags(**options)) + finally: + del ks + + def __richcmp__(self, other, int op): + if op == 2: # == + if other is self: + return True + elif not isinstance(other, _Trie): + return False + + return (<_Trie>self)._equals(other) + elif op == 3: # != + return not (self == other) + + raise TypeError("unorderable types: {0} and {1}".format( + self.__class__, other.__class__)) + + cdef bint _equals(self, _Trie other) nogil: + cdef int num_keys = self._trie.num_keys() + cdef base.NodeOrder node_order = self._trie.node_order() + if (other._trie.num_keys() != num_keys or + other._trie.node_order() != node_order): + return False + + cdef agent.Agent ag1, ag2 + ag1.set_query(b"") + ag2.set_query(b"") + cdef int i + cdef key.Key key1, key2 + for i in range(num_keys): + self._trie.predictive_search(ag1) + other._trie.predictive_search(ag2) + key1 = ag1.key() + key2 = ag2.key() + if (key1.length() != key2.length() or + strncmp(key1.ptr(), key2.ptr(), key1.length()) != 0): + return False + return True + + def __iter__(self): + return self.iterkeys() + + def __len__(self): + return self._trie.num_keys() + + def __contains__(self, key): + cdef bytes _key = self._encode_key(key) + return self._contains(_key) + + cdef bint _contains(self, bytes key): + cdef agent.Agent ag + ag.set_query(key, len(key)) + return self._trie.lookup(ag) + + def read(self, f): + """Read a trie from an open file. + + :param file f: a "real" on-disk file object. Passing a *file-like* + object would result in an error. + + .. deprecated:: 0.7.3 + + The method will be removed in version 0.8.0. Please use + :meth:`load` instead. + """ + warnings.warn("Trie.read is deprecated and will " + "be removed in marisa_trie 0.8.0. Please use " + "Trie.load instead.", DeprecationWarning) + self._trie.read(f.fileno()) + return self + + def write(self, f): + """Write a trie to an open file. + + :param file f: a "real" on-disk file object. Passing a *file-like* + object would result in an error. + + .. deprecated:: 0.7.3 + + The method will be removed in version 0.8.0. Please use + :meth:`save` instead. + """ + warnings.warn("Trie.write is deprecated and will " + "be removed in marisa_trie 0.8.0. Please use " + "Trie.save instead.", DeprecationWarning) + self._trie.write(f.fileno()) + + def save(self, path): + """Save a trie to a specified path.""" + with open(path, 'w') as f: + self._trie.write(f.fileno()) + + def load(self, path): + """Load a trie from a specified path.""" + with open(path, 'r') as f: + self._trie.read(f.fileno()) + return self + + cpdef bytes tobytes(self) except +: + """Return raw trie content as bytes.""" + cdef stringstream stream + iostream.write((<ostream *> &stream)[0], self._trie[0]) + cdef bytes res = stream.str() + return res + + cpdef frombytes(self, bytes data) except +: + """Load a trie from raw bytes generated by :meth:`tobytes`.""" + cdef stringstream* stream = new stringstream(data) + try: + iostream.read((<istream *> stream)[0], self._trie) + finally: + del stream + return self + + def __reduce__(self): + return self.__class__, (), self.tobytes() + + __setstate__ = frombytes + + def mmap(self, path): + """Memory map the content of a trie stored in a file. + + This allows to query trie without loading it fully in memory. + """ + import sys + str_path = path.encode(sys.getfilesystemencoding()) + cdef char* c_path = str_path + self._trie.mmap(c_path) + return self + + def iterkeys(self, prefix=None): + """ + Return an iterator over trie keys starting with a given ``prefix``. + """ + cdef agent.Agent ag + cdef bytes b_prefix = b'' + if prefix is not None: + b_prefix = self._encode_key(prefix) + ag.set_query(b_prefix, len(b_prefix)) + + while self._trie.predictive_search(ag): + yield self._get_key(ag) + + cpdef list keys(self, prefix=None): + """Return a list of trie keys starting with a given ``prefix``.""" + # non-generator inlined version of iterkeys() + cdef list res = [] + cdef bytes b_prefix = b'' + if prefix is not None: + b_prefix = self._encode_key(prefix) + cdef agent.Agent ag + ag.set_query(b_prefix, len(b_prefix)) + + while self._trie.predictive_search(ag): + res.append(self._get_key(ag)) + + return res + + def has_keys_with_prefix(self, prefix=""): + """ + Return ``True`` if any key in the trie begins with ``prefix``. + + .. deprecated:: 0.7.3 + + The method will be removed in version 0.8.0. Please use + :meth:`iterkeys` instead. + """ + warnings.warn("Trie.has_keys_with_prefix is deprecated and will " + "be removed in marisa_trie 0.8.0. Please use " + "Trie.iterkeys instead.", DeprecationWarning) + + cdef agent.Agent ag + cdef bytes b_prefix = self._encode_key(prefix) + ag.set_query(b_prefix, len(b_prefix)) + return self._trie.predictive_search(ag) + + +cdef class BinaryTrie(_Trie): + """A trie mapping bytes keys to auto-generated unique IDs.""" + + # key_id method is not in _Trie because it won't work for BytesTrie + cpdef int key_id(self, bytes key) except -1: + """Return an ID generated for a given ``key``. + + :raises KeyError: if key is not present in this trie. + """ + cdef int res = self._key_id(key, len(key)) + if res == -1: + raise KeyError(key) + return res + + cdef int _key_id(self, char* key, int len): + cdef bint res + cdef agent.Agent ag + ag.set_query(key, len) + res = self._trie.lookup(ag) + if not res: + return -1 + return ag.key().id() + + cpdef restore_key(self, int index): + """Return a key corresponding to a given ID.""" + cdef agent.Agent ag + ag.set_query(index) + try: + self._trie.reverse_lookup(ag) + except KeyError: + raise KeyError(index) + return self._get_key(ag) + + def __getitem__(self, bytes key): + return self.key_id(key) + + def get(self, bytes key, default=None): + """ + Return an ID for a given ``key`` or ``default`` if ``key`` is + not present in this trie. + """ + cdef int res + + res = self._key_id(key, len(key)) + if res == -1: + return default + return res + + def iter_prefixes(self, bytes key): + """ + Return an iterator of all prefixes of a given key. + """ + cdef agent.Agent ag + ag.set_query(key, len(key)) + + while self._trie.common_prefix_search(ag): + yield self._get_key(ag) + + def prefixes(self, bytes key): + """ + Return a list with all prefixes of a given key. + """ + # this an inlined version of ``list(self.iter_prefixes(key))`` + + cdef list res = [] + cdef agent.Agent ag + ag.set_query(key, len(key)) + + while self._trie.common_prefix_search(ag): + res.append(self._get_key(ag)) + return res + + def items(self, bytes prefix=b""): + # inlined for speed + cdef list res = [] + cdef agent.Agent ag + ag.set_query(prefix, len(prefix)) + + while self._trie.predictive_search(ag): + res.append((self._get_key(ag), ag.key().id())) + + return res + + def iteritems(self, bytes prefix=b""): + """ + Return an iterator over items that have a prefix ``prefix``. + """ + cdef agent.Agent ag + ag.set_query(prefix, len(prefix)) + + while self._trie.predictive_search(ag): + yield self._get_key(ag), ag.key().id() + + +cdef class _UnicodeKeyedTrie(_Trie): + """ + MARISA-trie wrapper for unicode keys. + """ + cdef bytes _encode_key(self, key): + return key.encode('utf8') + + cdef _get_key(self, agent.Agent& ag): + return <unicode>_Trie._get_key(self, ag).decode('utf8') + + +cdef class Trie(_UnicodeKeyedTrie): + """A trie mapping unicode keys to auto-generated unique IDs.""" + + # key_id method is not in _Trie because it won't work for BytesTrie + cpdef int key_id(self, unicode key) except -1: + """Return an ID generated for a given ``key``. + + :raises KeyError: if key is not present in this trie. + """ + cdef bytes _key = <bytes>key.encode('utf8') + cdef int res = self._key_id(_key) + if res == -1: + raise KeyError(key) + return res + + def __getitem__(self, unicode key): + return self.key_id(key) + + def get(self, key, default=None): + """ + Return an ID for a given ``key`` or ``default`` if ``key`` is + not present in this trie. + """ + cdef bytes b_key + cdef int res + + if isinstance(key, unicode): + b_key = <bytes>(<unicode>key).encode('utf8') + else: + b_key = key + + res = self._key_id(b_key) + if res == -1: + return default + return res + + cpdef restore_key(self, int index): + """Return a key corresponding to a given ID.""" + cdef agent.Agent ag + ag.set_query(index) + try: + self._trie.reverse_lookup(ag) + except KeyError: + raise KeyError(index) + return self._get_key(ag) + + cdef int _key_id(self, char* key): + cdef bint res + cdef agent.Agent ag + ag.set_query(key) + res = self._trie.lookup(ag) + if not res: + return -1 + return ag.key().id() + + def iter_prefixes(self, unicode key): + """ + Return an iterator of all prefixes of a given key. + """ + cdef bytes b_key = <bytes>key.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_key) + + while self._trie.common_prefix_search(ag): + yield self._get_key(ag) + + def prefixes(self, unicode key): + """ + Return a list with all prefixes of a given key. + """ + # this an inlined version of ``list(self.iter_prefixes(key))`` + + cdef list res = [] + cdef bytes b_key = <bytes>key.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_key) + + while self._trie.common_prefix_search(ag): + res.append(self._get_key(ag)) + return res + + def iter_prefixes_with_ids(self, unicode key): + """ + Return an iterator of (prefix, id) pairs of all prefixes of a given key. + """ + cdef bytes b_key = <bytes>key.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_key, len(b_key)) + + while self._trie.common_prefix_search(ag): + yield (self._get_key(ag), ag.key().id()) + + def iteritems(self, unicode prefix=""): + """ + Return an iterator over items that have a prefix ``prefix``. + """ + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + yield self._get_key(ag), ag.key().id() + + def items(self, unicode prefix=""): + # inlined for speed + cdef list res = [] + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + res.append((self._get_key(ag), ag.key().id())) + + return res + + +# This symbol is not allowed in utf8 so it is safe to use +# as a separator between utf8-encoded string and binary payload. +# XXX: b'\xff' value changes sort order for BytesTrie and RecordTrie. +# See https://github.com/kmike/DAWG docs for a description of a similar issue. +cdef bytes _VALUE_SEPARATOR = b'\xff' + + +cdef class BytesTrie(_UnicodeKeyedTrie): + """A trie mapping unicode keys to lists of bytes objects. + + The mapping is implemented by appending binary values to UTF8-encoded + and storing the result in MARISA-trie. + """ + cdef bytes _b_value_separator + cdef unsigned char _c_value_separator + + def __init__(self, arg=None, bytes value_separator=_VALUE_SEPARATOR, + **options): + """ + ``arg`` must be an iterable of tuples (unicode_key, bytes_payload). + """ + super(BytesTrie, self).__init__() + + self._b_value_separator = value_separator + self._c_value_separator = <unsigned char>ord(value_separator) + + byte_keys = (self._raw_key(d[0], d[1]) for d in (arg or [])) + self._build(byte_keys, **options) + + cpdef bytes _raw_key(self, unicode key, bytes payload): + return key.encode('utf8') + self._b_value_separator + payload + + cdef bint _contains(self, bytes key): + cdef agent.Agent ag + cdef bytes _key = key + self._b_value_separator + ag.set_query(_key) + return self._trie.predictive_search(ag) + + cpdef list prefixes(self, unicode key): + """ + Return a list with all prefixes of a given key. + """ + + # XXX: is there a char-walking API in libmarisa? + # This implementation is suboptimal. + + cdef agent.Agent ag + cdef list res = [] + cdef int key_len = len(key) + cdef unicode prefix + cdef bytes b_prefix + cdef int ind = 1 + + while ind <= key_len: + prefix = key[:ind] + b_prefix = <bytes>(prefix.encode('utf8') + self._b_value_separator) + ag.set_query(b_prefix) + if self._trie.predictive_search(ag): + res.append(prefix) + + ind += 1 + + return res + + def __getitem__(self, key): + cdef list res = self.get(key) + if res is None: + raise KeyError(key) + return res + + cpdef get(self, key, default=None): + """ + Return a list of payloads (as byte objects) for a given key + or ``default`` if the key is not found. + """ + cdef list res + + if isinstance(key, unicode): + res = self.get_value(<unicode>key) + else: + res = self.b_get_value(key) + + if not res: + return default + return res + + cpdef list get_value(self, unicode key): + """ + Return a list of payloads (as byte objects) for a given unicode key. + """ + cdef bytes b_key = <bytes>key.encode('utf8') + return self.b_get_value(b_key) + + cpdef list b_get_value(self, bytes key): + """ + Return a list of payloads (as byte objects) for a given utf8-encoded key. + """ + cdef list res = [] + cdef bytes value + cdef bytes b_prefix = key + self._b_value_separator + cdef int prefix_len = len(b_prefix) + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + value = ag.key().ptr()[prefix_len:ag.key().length()] + res.append(value) + + return res + + cpdef list items(self, unicode prefix=""): + # copied from iteritems for speed + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef bytes value + cdef unicode key + cdef unsigned char* raw_key + cdef list res = [] + cdef int i, value_len + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + break + + key = raw_key[:i].decode('utf8') + value = raw_key[i+1:ag.key().length()] + + res.append( + (key, value) + ) + return res + + def iteritems(self, unicode prefix=""): + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef bytes value + cdef unicode key + cdef unsigned char* raw_key + cdef int i, value_len + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + break + + key = raw_key[:i].decode('utf8') + value = raw_key[i+1:ag.key().length()] + + yield key, value + + cpdef list keys(self, prefix=""): + # copied from iterkeys for speed + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef unicode key + cdef unsigned char* raw_key + cdef list res = [] + cdef int i + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + key = raw_key[:i].decode('utf8') + res.append(key) + break + return res + + def iterkeys(self, unicode prefix=""): + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef unicode key + cdef unsigned char* raw_key + cdef int i + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + yield raw_key[:i].decode('utf8') + break + + +cdef class _UnpackTrie(BytesTrie): + + def __init__(self, arg=None, **options): + keys = ((d[0], self._pack(d[1])) for d in (arg or [])) + super(_UnpackTrie, self).__init__(keys, **options) + + cdef _unpack(self, bytes value): + return value + + cdef bytes _pack(self, value): + return value + + cpdef list b_get_value(self, bytes key): + cdef list values = BytesTrie.b_get_value(self, key) + return [self._unpack(val) for val in values] + + cpdef list items(self, unicode prefix=""): + cdef list items = BytesTrie.items(self, prefix) + return [(key, self._unpack(val)) for (key, val) in items] + + def iteritems(self, unicode prefix=""): + return ((key, self._unpack(val)) for key, val in BytesTrie.iteritems(self, prefix)) + + +cdef class RecordTrie(_UnpackTrie): + """A trie mapping unicode keys to lists of data tuples. + + The data is packed using :mod:`struct` module, therefore all + tuples must be of the same format. See :mod:`struct` documentation + for available format strings. + + The mapping is implemented by appending binary values to UTF8-encoded + and storing the result in MARISA-trie. + """ + cdef _struct + cdef _fmt + + def __init__(self, fmt, arg=None, **options): + """ + ``arg`` must be an iterable of tuples (unicode_key, data_tuple). + Data tuples will be converted to bytes with + ``struct.pack(fmt, *data_tuple)``. + """ + self._fmt = fmt + self._struct = struct.Struct(str(fmt)) + super(RecordTrie, self).__init__(arg, **options) + + cdef _unpack(self, bytes value): + return self._struct.unpack(value) + + cdef bytes _pack(self, value): + return self._struct.pack(*value) + + def __reduce__(self): + return self.__class__, (self._fmt, ), self.tobytes() diff --git a/contrib/python/marisa-trie/py2/src/query.pxd b/contrib/python/marisa-trie/py2/src/query.pxd new file mode 100644 index 00000000000..a650bb89658 --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/query.pxd @@ -0,0 +1,20 @@ +cdef extern from "<marisa/query.h>" namespace "marisa" nogil: + + cdef cppclass Query: + Query() + Query(Query &query) + + #Query &operator=(Query &query) + + char operator[](int i) + + void set_str(char *str) + void set_str(char *ptr, int length) + void set_id(int id) + + char *ptr() + int length() + int id() + + void clear() + void swap(Query &rhs) diff --git a/contrib/python/marisa-trie/py2/src/std_iostream.pxd b/contrib/python/marisa-trie/py2/src/std_iostream.pxd new file mode 100644 index 00000000000..bf7d0e89aa7 --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/std_iostream.pxd @@ -0,0 +1,18 @@ +from libcpp.string cimport string + +cdef extern from "<istream>" namespace "std" nogil: + cdef cppclass istream: + istream() except + + istream& read (char* s, int n) except + + + cdef cppclass ostream: + ostream() except + + ostream& write (char* s, int n) except + + +cdef extern from "<sstream>" namespace "std" nogil: + + cdef cppclass stringstream: + stringstream() + stringstream(string s) + string str () + diff --git a/contrib/python/marisa-trie/py2/src/trie.pxd b/contrib/python/marisa-trie/py2/src/trie.pxd new file mode 100644 index 00000000000..f525caf8ad6 --- /dev/null +++ b/contrib/python/marisa-trie/py2/src/trie.pxd @@ -0,0 +1,41 @@ +cimport agent +cimport base +cimport keyset + + +cdef extern from "<marisa/trie.h>" namespace "marisa" nogil: + + cdef cppclass Trie: + Trie() + + void build(keyset.Keyset &keyset, int config_flags) except + + void build(keyset.Keyset &keyset) except + + + void mmap(char *filename) except + + void map(void *ptr, int size) except + + + void load(char *filename) except + + void read(int fd) except + + + void save(char *filename) except + + void write(int fd) except + + + bint lookup(agent.Agent &agent) except + + void reverse_lookup(agent.Agent &agent) except +KeyError + bint common_prefix_search(agent.Agent &agent) except + + bint predictive_search(agent.Agent &agent) except + + + int num_tries() except + + int num_keys() except + + int num_nodes() except + + + base.TailMode tail_mode() + base.NodeOrder node_order() + + bint empty() except + + int size() except + + int total_size() except + + int io_size() except + + + void clear() except + + void swap(Trie &rhs) except + diff --git a/contrib/python/marisa-trie/py2/ya.make b/contrib/python/marisa-trie/py2/ya.make new file mode 100644 index 00000000000..d94b0d01541 --- /dev/null +++ b/contrib/python/marisa-trie/py2/ya.make @@ -0,0 +1,50 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +PROVIDES(MARISA-TRIE) + +SUBSCRIBER(g:python-contrib) + +VERSION(0.8.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/setuptools +) + +ADDINCL( + contrib/python/marisa-trie/py2/marisa-trie/include + contrib/python/marisa-trie/py2/marisa-trie/lib +) + +NO_COMPILER_WARNINGS() + +NO_LINT() + +SRCS( + marisa-trie/lib/marisa/agent.cc + marisa-trie/lib/marisa/grimoire/io/mapper.cc + marisa-trie/lib/marisa/grimoire/io/reader.cc + marisa-trie/lib/marisa/grimoire/io/writer.cc + marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc + marisa-trie/lib/marisa/grimoire/trie/tail.cc + marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc + marisa-trie/lib/marisa/keyset.cc + marisa-trie/lib/marisa/trie.cc +) + +PY_SRCS( + TOP_LEVEL + CYTHON_CPP + src/marisa_trie.pyx=marisa_trie +) + +RESOURCE_FILES( + PREFIX contrib/python/marisa-trie/py2/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/marisa-trie/py3/.dist-info/METADATA b/contrib/python/marisa-trie/py3/.dist-info/METADATA new file mode 100644 index 00000000000..8d866d15b8b --- /dev/null +++ b/contrib/python/marisa-trie/py3/.dist-info/METADATA @@ -0,0 +1,314 @@ +Metadata-Version: 2.1 +Name: marisa-trie +Version: 1.2.1 +Summary: Static memory-efficient and fast Trie-like structures for Python. +Home-page: https://github.com/pytries/marisa-trie +Author: Mikhail Korobov +Author-email: [email protected] +License: MIT +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Scientific/Engineering :: Information Analysis +Classifier: Topic :: Text Processing :: Linguistic +Requires-Python: >=3.7 +Description-Content-Type: text/x-rst +License-File: LICENSE +License-File: AUTHORS.rst +Requires-Dist: setuptools +Provides-Extra: test +Requires-Dist: hypothesis; extra == "test" +Requires-Dist: pytest; extra == "test" +Requires-Dist: readme-renderer; extra == "test" + +MARISA Trie +=========== + +.. image:: https://img.shields.io/pypi/pyversions/marisa-trie.svg + :target: https://pypi.python.org/pypi/marisa-trie + +.. image:: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml/badge.svg + :target: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml + +Static memory-efficient Trie-like structures for Python (3.7+) +based on `marisa-trie`_ C++ library. + +String data in a MARISA-trie may take up to 50x-100x less memory than +in a standard Python dict; the raw lookup speed is comparable; trie also +provides fast advanced methods like prefix search. + +.. note:: + + There are official SWIG-based Python bindings included + in C++ library distribution; this package provides alternative + Cython-based pip-installable Python bindings. + +.. _marisa-trie: https://github.com/s-yata/marisa-trie + +Installation +============ + +:: + + python -m pip install -U marisa-trie + +Usage +===== + +See `tutorial`_ and `API`_ for details. + +.. _tutorial: https://marisa-trie.readthedocs.io/en/latest/tutorial.html +.. _API: https://marisa-trie.readthedocs.io/en/latest/api.html + +Current limitations +=================== + +* The library is not tested with mingw32 compiler; +* ``.prefixes()`` method of ``BytesTrie`` and ``RecordTrie`` is quite slow + and doesn't have iterator counterpart; +* ``read()`` and ``write()`` methods don't work with file-like objects + (they work only with real files; pickling works fine for file-like objects); +* there are ``keys()`` and ``items()`` methods but no ``values()`` method. + +License +======= + +Wrapper code is licensed under MIT License. + +Bundled `marisa-trie`_ C++ library is dual-licensed under +LGPL and BSD 2-clause license. + +CHANGES +======= + +1.2.1 (2024-10-12) +------------------ + +* Publish Python 3.13 wheels (only CPython ones, PyPy ones are skipped until https://github.com/pypa/distutils/issues/283 is fixed). +* Rebuild Cython wrapper with Cython 3.0.11. + +1.2.0 (2024-06-05) +------------------ + +* Added Python 3.13 support (#105). +* Rebuild Cython wrapper with Cython 3.0.10 (#105). + +1.1.1 (2024-05-06) +------------------ + +* Publish Linux aarch64 wheels (#101). + +1.1.0 (2023-10-06) +------------------ + +* Added Python 3.12 support. + +1.0.0 (2023-09-03) +------------------ + +* Dropped Python 2.7, 3.4, 3.5, 3.6 support. +* Added ``Trie.map()`` (#90). +* Rebuilt Cython wrapper with Cython 3.0.2. +* Fixed benchmark documentation typos (#89). + +0.8.0 (2023-03-25) +------------------ + +* Add ``Trie.iter_prefixes_with_ids()`` method to return ``(prefix, id)`` pairs (#83). +* Rebuild Cython wrapper with Cython 0.29.33 (#88). + +0.7.8 (2022-10-25) +------------------ + +* Added Python 3.11 support. +* Rebuild Cython wrapper with Cython 0.29.32. + +0.7.7 (2021-08-04) +------------------ + +* Restored Python 2.7 support. +* Fixed README image references not working on Windows. + +0.7.6 (2021-07-28) +------------------ + +* Wheels are now published for all platforms. +* Fixed ``ResourceWarning: unclosed file`` in ``setup.py``. +* Run ``black`` on the entire source code. +* Moved the QA/CI to GitHub. +* Rebuild Cython wrapper with Cython 0.29.24. +* Updated ``libmarisa-trie`` to the latest version (0.2.6). +* Fixed failing tests and usage of deprecated methods. +* Expanded supported Python version (2.7, 3.4 - 3.10). + +0.7.5 (2018-04-10) +------------------ + +* Removed redundant ``DeprecationWarning`` messages in ``Trie.save`` and + ``Trie.load``. +* Dropped support for Python 2.6. +* Rebuild Cython wrapper with Cython 0.28.1. + +0.7.4 (2017-03-27) +------------------ + +* Fixed packaging issue, ``MANIFEST.in`` was not updated after ``libmarisa-trie`` + became a submodule. + +0.7.3 (2017-02-14) +------------------ + +* Added ``BinaryTrie`` for storing arbitrary sequences of bytes, e.g. IP + addresses (thanks Tomasz Melcer); +* Deprecated ``Trie.has_keys_with_prefix`` which can be trivially implemented in + terms of ``Trie.iterkeys``; +* Deprecated ``Trie.read`` and ``Trie.write`` which onlywork for "real" files + and duplicate the functionality of ``load`` and ``save``. See issue #31 on + GitHub; +* Updated ``libmarisa-trie`` to the latest version. Yay, 64-bit Windows support. +* Rebuilt Cython wrapper with Cython 0.25.2. + +0.7.2 (2015-04-21) +------------------ + +* packaging issue is fixed. + +0.7.1 (2015-04-21) +------------------ + +* setup.py is switched to setuptools; +* a tiny speedup; +* wrapper is rebuilt with Cython 0.22. + +0.7 (2014-12-15) +---------------- + +* ``trie1 == trie2`` and ``trie1 != trie2`` now work (thanks Sergei Lebedev); +* ``for key in trie:`` is fixed (thanks Sergei Lebedev); +* wrapper is rebuilt with Cython 0.21.1 (thanks Sergei Lebedev); +* https://bitbucket.org/kmike/marisa-trie repo is no longer supported. + +0.6 (2014-02-22) +---------------- + +* New ``Trie`` methods: ``__getitem__``, ``get``, ``items``, ``iteritems``. + ``trie[u'key']`` is now the same as ``trie.key_id(u'key')``. +* small optimization for ``BytesTrie.get``. +* wrapper is rebuilt with Cython 0.20.1. + +0.5.3 (2014-02-08) +------------------ + +* small ``Trie.restore_key`` optimization (it should work 5-15% faster) + +0.5.2 (2014-02-08) +------------------ + +* fix ``Trie.restore_key`` method - it was reading past declared string length; +* rebuild wrapper with Cython 0.20. + +0.5.1 (2013-10-03) +------------------ + +* ``has_keys_with_prefix(prefix)`` method (thanks + `Matt Hickford <https://github.com/matt-hickford>`_) + +0.5 (2013-05-07) +---------------- + +* ``BytesTrie.iterkeys``, ``BytesTrie.iteritems``, + ``RecordTrie.iterkeys`` and ``RecordTrie.iteritems`` methods; +* wrapper is rebuilt with Cython 0.19; +* ``value_separator`` parameter for ``BytesTrie`` and ``RecordTrie``. + +0.4 (2013-02-28) +---------------- + +* improved trie building: ``weights`` optional parameter; +* improved trie building: unnecessary input sorting is removed; +* wrapper is rebuilt with Cython 0.18; +* bundled marisa-trie C++ library is updated to svn r133. + +0.3.8 (2013-01-03) +------------------ + +* Rebuild wrapper with Cython pre-0.18; +* update benchmarks. + +0.3.7 (2012-09-21) +------------------ + +* Update bundled marisa-trie C++ library (this may fix more mingw issues); +* Python 3.3 support is back. + +0.3.6 (2012-09-05) +------------------ + +* much faster (3x-7x) ``.items()`` and ``.keys()`` methods for all tries; + faster (up to 3x) ``.prefixes()`` method for ``Trie``. + +0.3.5 (2012-08-30) +------------------ + +* Pickling of RecordTrie is fixed (thanks lazarou for the report); +* error messages should become more useful. + +0.3.4 (2012-08-29) +------------------ + +* Issues with mingw32 should be resolved (thanks Susumu Yata). + +0.3.3 (2012-08-27) +------------------ + +* ``.get(key, default=None)`` method for ``BytesTrie`` and ``RecordTrie``; +* small README improvements. + +0.3.2 (2012-08-26) +------------------ + +* Small code cleanup; +* ``load``, ``read`` and ``mmap`` methods returns 'self'; +* I can't run tests (via tox) under Python 3.3 so it is + removed from supported versions for now. + +0.3.1 (2012-08-23) +------------------ + +* ``.prefixes()`` support for RecordTrie and BytesTrie. + +0.3 (2012-08-23) +---------------- + +* RecordTrie and BytesTrie are introduced; +* IntTrie class is removed (probably temporary?); +* dumps/loads methods are renamed to tobytes/frombytes; +* benchmark & tests improvements; +* support for MARISA-trie config options is added. + +0.2 (2012-08-19) +------------------ + +* Pickling/unpickling support; +* dumps/loads methods; +* python 3.3 workaround; +* improved tests; +* benchmarks. + +0.1 (2012-08-17) +---------------- + +Initial release. diff --git a/contrib/python/marisa-trie/py3/.dist-info/top_level.txt b/contrib/python/marisa-trie/py3/.dist-info/top_level.txt new file mode 100644 index 00000000000..62049767090 --- /dev/null +++ b/contrib/python/marisa-trie/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +marisa_trie diff --git a/contrib/python/marisa-trie/py3/.yandex_meta/yamaker.yaml b/contrib/python/marisa-trie/py3/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..bfd4439ead4 --- /dev/null +++ b/contrib/python/marisa-trie/py3/.yandex_meta/yamaker.yaml @@ -0,0 +1,5 @@ +cython_directive: + - language_level=3 +copy: + - marisa-trie/* + - src/* diff --git a/contrib/python/marisa-trie/py3/AUTHORS.rst b/contrib/python/marisa-trie/py3/AUTHORS.rst new file mode 100644 index 00000000000..9df75381ed6 --- /dev/null +++ b/contrib/python/marisa-trie/py3/AUTHORS.rst @@ -0,0 +1,7 @@ +Authors and contributors, in no particular order: + +* Mikhail Korobov <[email protected]> +* `Matt Hickford <https://github.com/matt-hickford>`_ +* Sergei Lebedev <[email protected]> +* Tomasz Melcer <https://github.com/liori> +* `Mickaël Schoentgen <https://github.com/BoboTiG>` diff --git a/contrib/python/marisa-trie/py3/LICENSE b/contrib/python/marisa-trie/py3/LICENSE new file mode 100644 index 00000000000..e65b3446adb --- /dev/null +++ b/contrib/python/marisa-trie/py3/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) marisa-trie authors and contributors, 2012-2024 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR +A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/contrib/python/marisa-trie/py3/README.rst b/contrib/python/marisa-trie/py3/README.rst new file mode 100644 index 00000000000..0f123a32860 --- /dev/null +++ b/contrib/python/marisa-trie/py3/README.rst @@ -0,0 +1,56 @@ +MARISA Trie +=========== + +.. image:: https://img.shields.io/pypi/pyversions/marisa-trie.svg + :target: https://pypi.python.org/pypi/marisa-trie + +.. image:: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml/badge.svg + :target: https://github.com/pytries/marisa-trie/actions/workflows/tests.yml + +Static memory-efficient Trie-like structures for Python (3.7+) +based on `marisa-trie`_ C++ library. + +String data in a MARISA-trie may take up to 50x-100x less memory than +in a standard Python dict; the raw lookup speed is comparable; trie also +provides fast advanced methods like prefix search. + +.. note:: + + There are official SWIG-based Python bindings included + in C++ library distribution; this package provides alternative + Cython-based pip-installable Python bindings. + +.. _marisa-trie: https://github.com/s-yata/marisa-trie + +Installation +============ + +:: + + python -m pip install -U marisa-trie + +Usage +===== + +See `tutorial`_ and `API`_ for details. + +.. _tutorial: https://marisa-trie.readthedocs.io/en/latest/tutorial.html +.. _API: https://marisa-trie.readthedocs.io/en/latest/api.html + +Current limitations +=================== + +* The library is not tested with mingw32 compiler; +* ``.prefixes()`` method of ``BytesTrie`` and ``RecordTrie`` is quite slow + and doesn't have iterator counterpart; +* ``read()`` and ``write()`` methods don't work with file-like objects + (they work only with real files; pickling works fine for file-like objects); +* there are ``keys()`` and ``items()`` methods but no ``values()`` method. + +License +======= + +Wrapper code is licensed under MIT License. + +Bundled `marisa-trie`_ C++ library is dual-licensed under +LGPL and BSD 2-clause license. diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/agent.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/agent.h new file mode 100644 index 00000000000..c72dea755d4 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/agent.h @@ -0,0 +1,73 @@ +#ifndef MARISA_AGENT_H_ +#define MARISA_AGENT_H_ + +#include "marisa/key.h" +#include "marisa/query.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class State; + +} // namespace trie +} // namespace grimoire + +class Agent { + public: + Agent(); + ~Agent(); + + const Query &query() const { + return query_; + } + const Key &key() const { + return key_; + } + + void set_query(const char *str); + void set_query(const char *ptr, std::size_t length); + void set_query(std::size_t key_id); + + const grimoire::trie::State &state() const { + return *state_; + } + grimoire::trie::State &state() { + return *state_; + } + + void set_key(const char *str) { + MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR); + key_.set_str(str); + } + void set_key(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_.set_str(ptr, length); + } + void set_key(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_.set_id(id); + } + + bool has_state() const { + return state_.get() != NULL; + } + void init_state(); + + void clear(); + void swap(Agent &rhs); + + private: + Query query_; + Key key_; + scoped_ptr<grimoire::trie::State> state_; + + // Disallows copy and assignment. + Agent(const Agent &); + Agent &operator=(const Agent &); +}; + +} // namespace marisa + +#endif // MARISA_AGENT_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/base.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/base.h new file mode 100644 index 00000000000..ffcdc5bd8cc --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/base.h @@ -0,0 +1,196 @@ +#ifndef MARISA_BASE_H_ +#define MARISA_BASE_H_ + +// Old Visual C++ does not provide stdint.h. +#ifndef _MSC_VER + #include <stdint.h> +#endif // _MSC_VER + +#ifdef __cplusplus + #include <cstddef> +#else // __cplusplus + #include <stddef.h> +#endif // __cplusplus + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#ifdef _MSC_VER +typedef unsigned __int8 marisa_uint8; +typedef unsigned __int16 marisa_uint16; +typedef unsigned __int32 marisa_uint32; +typedef unsigned __int64 marisa_uint64; +#else // _MSC_VER +typedef uint8_t marisa_uint8; +typedef uint16_t marisa_uint16; +typedef uint32_t marisa_uint32; +typedef uint64_t marisa_uint64; +#endif // _MSC_VER + +#if defined(_WIN64) || defined(__amd64__) || defined(__x86_64__) || \ + defined(__ia64__) || defined(__ppc64__) || defined(__powerpc64__) || \ + defined(__sparc64__) || defined(__mips64__) || defined(__aarch64__) || \ + defined(__s390x__) + #define MARISA_WORD_SIZE 64 +#else // defined(_WIN64), etc. + #define MARISA_WORD_SIZE 32 +#endif // defined(_WIN64), etc. + +//#define MARISA_WORD_SIZE (sizeof(void *) * 8) + +#define MARISA_UINT8_MAX ((marisa_uint8)~(marisa_uint8)0) +#define MARISA_UINT16_MAX ((marisa_uint16)~(marisa_uint16)0) +#define MARISA_UINT32_MAX ((marisa_uint32)~(marisa_uint32)0) +#define MARISA_UINT64_MAX ((marisa_uint64)~(marisa_uint64)0) +#define MARISA_SIZE_MAX ((size_t)~(size_t)0) + +#define MARISA_INVALID_LINK_ID MARISA_UINT32_MAX +#define MARISA_INVALID_KEY_ID MARISA_UINT32_MAX +#define MARISA_INVALID_EXTRA (MARISA_UINT32_MAX >> 8) + +// Error codes are defined as members of marisa_error_code. This library throws +// an exception with one of the error codes when an error occurs. +typedef enum marisa_error_code_ { + // MARISA_OK means that a requested operation has succeeded. In practice, an + // exception never has MARISA_OK because it is not an error. + MARISA_OK = 0, + + // MARISA_STATE_ERROR means that an object was not ready for a requested + // operation. For example, an operation to modify a fixed vector throws an + // exception with MARISA_STATE_ERROR. + MARISA_STATE_ERROR = 1, + + // MARISA_NULL_ERROR means that an invalid NULL pointer has been given. + MARISA_NULL_ERROR = 2, + + // MARISA_BOUND_ERROR means that an operation has tried to access an out of + // range address. + MARISA_BOUND_ERROR = 3, + + // MARISA_RANGE_ERROR means that an out of range value has appeared in + // operation. + MARISA_RANGE_ERROR = 4, + + // MARISA_CODE_ERROR means that an undefined code has appeared in operation. + MARISA_CODE_ERROR = 5, + + // MARISA_RESET_ERROR means that a smart pointer has tried to reset itself. + MARISA_RESET_ERROR = 6, + + // MARISA_SIZE_ERROR means that a size has exceeded a library limitation. + MARISA_SIZE_ERROR = 7, + + // MARISA_MEMORY_ERROR means that a memory allocation has failed. + MARISA_MEMORY_ERROR = 8, + + // MARISA_IO_ERROR means that an I/O operation has failed. + MARISA_IO_ERROR = 9, + + // MARISA_FORMAT_ERROR means that input was in invalid format. + MARISA_FORMAT_ERROR = 10, +} marisa_error_code; + +// Min/max values, flags and masks for dictionary settings are defined below. +// Please note that unspecified settings will be replaced with the default +// settings. For example, 0 is equivalent to (MARISA_DEFAULT_NUM_TRIES | +// MARISA_DEFAULT_TRIE | MARISA_DEFAULT_TAIL | MARISA_DEFAULT_ORDER). + +// A dictionary consists of 3 tries in default. Usually more tries make a +// dictionary space-efficient but time-inefficient. +typedef enum marisa_num_tries_ { + MARISA_MIN_NUM_TRIES = 0x00001, + MARISA_MAX_NUM_TRIES = 0x0007F, + MARISA_DEFAULT_NUM_TRIES = 0x00003, +} marisa_num_tries; + +// This library uses a cache technique to accelerate search functions. The +// following enumerated type marisa_cache_level gives a list of available cache +// size options. A larger cache enables faster search but takes a more space. +typedef enum marisa_cache_level_ { + MARISA_HUGE_CACHE = 0x00080, + MARISA_LARGE_CACHE = 0x00100, + MARISA_NORMAL_CACHE = 0x00200, + MARISA_SMALL_CACHE = 0x00400, + MARISA_TINY_CACHE = 0x00800, + MARISA_DEFAULT_CACHE = MARISA_NORMAL_CACHE +} marisa_cache_level; + +// This library provides 2 kinds of TAIL implementations. +typedef enum marisa_tail_mode_ { + // MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is + // available if and only if the last labels do not contain a NULL character. + // If MARISA_TEXT_TAIL is specified and a NULL character exists in the last + // labels, the setting is automatically switched to MARISA_BINARY_TAIL. + MARISA_TEXT_TAIL = 0x01000, + + // MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses + // a bit vector to detect the end of a sequence, instead of NULL characters. + // So, MARISA_BINARY_TAIL requires a larger space if the average length of + // labels is greater than 8. + MARISA_BINARY_TAIL = 0x02000, + + MARISA_DEFAULT_TAIL = MARISA_TEXT_TAIL, +} marisa_tail_mode; + +// The arrangement of nodes affects the time cost of matching and the order of +// predictive search. +typedef enum marisa_node_order_ { + // MARISA_LABEL_ORDER arranges nodes in ascending label order. + // MARISA_LABEL_ORDER is useful if an application needs to predict keys in + // label order. + MARISA_LABEL_ORDER = 0x10000, + + // MARISA_WEIGHT_ORDER arranges nodes in descending weight order. + // MARISA_WEIGHT_ORDER is generally a better choice because it enables faster + // matching. + MARISA_WEIGHT_ORDER = 0x20000, + + MARISA_DEFAULT_ORDER = MARISA_WEIGHT_ORDER, +} marisa_node_order; + +typedef enum marisa_config_mask_ { + MARISA_NUM_TRIES_MASK = 0x0007F, + MARISA_CACHE_LEVEL_MASK = 0x00F80, + MARISA_TAIL_MODE_MASK = 0x0F000, + MARISA_NODE_ORDER_MASK = 0xF0000, + MARISA_CONFIG_MASK = 0xFFFFF +} marisa_config_mask; + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#ifdef __cplusplus + +// `std::swap` is in <utility> since C++ 11 but in <algorithm> in C++ 98: +#if __cplusplus >= 201103L + #include <utility> +#else + #include <algorithm> +#endif +namespace marisa { + +typedef ::marisa_uint8 UInt8; +typedef ::marisa_uint16 UInt16; +typedef ::marisa_uint32 UInt32; +typedef ::marisa_uint64 UInt64; + +typedef ::marisa_error_code ErrorCode; + +typedef ::marisa_cache_level CacheLevel; +typedef ::marisa_tail_mode TailMode; +typedef ::marisa_node_order NodeOrder; + +using std::swap; + +} // namespace marisa +#endif // __cplusplus + +#ifdef __cplusplus + #include "marisa/exception.h" + #include "marisa/scoped-ptr.h" + #include "marisa/scoped-array.h" +#endif // __cplusplus + +#endif // MARISA_BASE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/exception.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/exception.h new file mode 100644 index 00000000000..c4a93aa4978 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/exception.h @@ -0,0 +1,82 @@ +#ifndef MARISA_EXCEPTION_H_ +#define MARISA_EXCEPTION_H_ + +#include <exception> + +#include "marisa/base.h" + +namespace marisa { + +// An exception object keeps a filename, a line number, an error code and an +// error message. The message format is as follows: +// "__FILE__:__LINE__: error_code: error_message" +class Exception : public std::exception { + public: + Exception(const char *filename, int line, + ErrorCode error_code, const char *error_message) + : std::exception(), filename_(filename), line_(line), + error_code_(error_code), error_message_(error_message) {} + Exception(const Exception &ex) + : std::exception(), filename_(ex.filename_), line_(ex.line_), + error_code_(ex.error_code_), error_message_(ex.error_message_) {} + virtual ~Exception() {} + + Exception &operator=(const Exception &rhs) { + filename_ = rhs.filename_; + line_ = rhs.line_; + error_code_ = rhs.error_code_; + error_message_ = rhs.error_message_; + return *this; + } + + const char *filename() const { + return filename_; + } + int line() const { + return line_; + } + ErrorCode error_code() const { + return error_code_; + } + const char *error_message() const { + return error_message_; + } + + virtual const char *what() const noexcept { + return error_message_; + } + + private: + const char *filename_; + int line_; + ErrorCode error_code_; + const char *error_message_; +}; + +// These macros are used to convert a line number to a string constant. +#define MARISA_INT_TO_STR(value) #value +#define MARISA_LINE_TO_STR(line) MARISA_INT_TO_STR(line) +#define MARISA_LINE_STR MARISA_LINE_TO_STR(__LINE__) + +// MARISA_THROW throws an exception with a filename, a line number, an error +// code and an error message. The message format is as follows: +// "__FILE__:__LINE__: error_code: error_message" +#define MARISA_THROW(error_code, error_message) \ + (throw marisa::Exception(__FILE__, __LINE__, error_code, \ + __FILE__ ":" MARISA_LINE_STR ": " #error_code ": " error_message)) + +// MARISA_THROW_IF throws an exception if `condition' is true. +#define MARISA_THROW_IF(condition, error_code) \ + (void)((!(condition)) || (MARISA_THROW(error_code, #condition), 0)) + +// MARISA_DEBUG_IF is ignored if _DEBUG is undefined. So, it is useful for +// debugging time-critical codes. +#ifdef _DEBUG + #define MARISA_DEBUG_IF(cond, error_code) MARISA_THROW_IF(cond, error_code) +#else + #define MARISA_DEBUG_IF(cond, error_code) +#endif + +} // namespace marisa + +#endif // MARISA_EXCEPTION_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/iostream.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/iostream.h new file mode 100644 index 00000000000..05d139a23b8 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/iostream.h @@ -0,0 +1,18 @@ +#ifndef MARISA_IOSTREAM_H_ +#define MARISA_IOSTREAM_H_ + +#include <iosfwd> + +namespace marisa { + +class Trie; + +std::istream &read(std::istream &stream, Trie *trie); +std::ostream &write(std::ostream &stream, const Trie &trie); + +std::istream &operator>>(std::istream &stream, Trie &trie); +std::ostream &operator<<(std::ostream &stream, const Trie &trie); + +} // namespace marisa + +#endif // MARISA_IOSTREAM_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/key.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/key.h new file mode 100644 index 00000000000..8b2cf8fcf8a --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/key.h @@ -0,0 +1,85 @@ +#ifndef MARISA_KEY_H_ +#define MARISA_KEY_H_ + +#include "marisa/base.h" + +namespace marisa { + +class Key { + public: + Key() : ptr_(NULL), length_(0), union_() { + union_.id = 0; + } + Key(const Key &key) + : ptr_(key.ptr_), length_(key.length_), union_(key.union_) {} + + Key &operator=(const Key &key) { + ptr_ = key.ptr_; + length_ = key.length_; + union_ = key.union_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return ptr_[i]; + } + + void set_str(const char *str) { + MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR); + std::size_t length = 0; + while (str[length] != '\0') { + ++length; + } + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = str; + length_ = (UInt32)length; + } + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr; + length_ = (UInt32)length; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + union_.id = (UInt32)id; + } + void set_weight(float weight) { + union_.weight = weight; + } + + const char *ptr() const { + return ptr_; + } + std::size_t length() const { + return length_; + } + std::size_t id() const { + return union_.id; + } + float weight() const { + return union_.weight; + } + + void clear() { + Key().swap(*this); + } + void swap(Key &rhs) { + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(length_, rhs.length_); + marisa::swap(union_.id, rhs.union_.id); + } + + private: + const char *ptr_; + UInt32 length_; + union Union { + UInt32 id; + float weight; + } union_; +}; + +} // namespace marisa + +#endif // MARISA_KEY_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/keyset.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/keyset.h new file mode 100644 index 00000000000..e575015c0c3 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/keyset.h @@ -0,0 +1,80 @@ +#ifndef MARISA_KEYSET_H_ +#define MARISA_KEYSET_H_ + +#include "marisa/key.h" + +namespace marisa { + +class Keyset { + public: + enum { + BASE_BLOCK_SIZE = 4096, + EXTRA_BLOCK_SIZE = 1024, + KEY_BLOCK_SIZE = 256 + }; + + Keyset(); + + void push_back(const Key &key); + void push_back(const Key &key, char end_marker); + + void push_back(const char *str); + void push_back(const char *ptr, std::size_t length, float weight = 1.0); + + const Key &operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return key_blocks_[i / KEY_BLOCK_SIZE][i % KEY_BLOCK_SIZE]; + } + Key &operator[](std::size_t i) { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return key_blocks_[i / KEY_BLOCK_SIZE][i % KEY_BLOCK_SIZE]; + } + + std::size_t num_keys() const { + return size_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + std::size_t total_length() const { + return total_length_; + } + + void reset(); + + void clear(); + void swap(Keyset &rhs); + + private: + scoped_array<scoped_array<char> > base_blocks_; + std::size_t base_blocks_size_; + std::size_t base_blocks_capacity_; + scoped_array<scoped_array<char> > extra_blocks_; + std::size_t extra_blocks_size_; + std::size_t extra_blocks_capacity_; + scoped_array<scoped_array<Key> > key_blocks_; + std::size_t key_blocks_size_; + std::size_t key_blocks_capacity_; + char *ptr_; + std::size_t avail_; + std::size_t size_; + std::size_t total_length_; + + char *reserve(std::size_t size); + + void append_base_block(); + void append_extra_block(std::size_t size); + void append_key_block(); + + // Disallows copy and assignment. + Keyset(const Keyset &); + Keyset &operator=(const Keyset &); +}; + +} // namespace marisa + +#endif // MARISA_KEYSET_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/query.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/query.h new file mode 100644 index 00000000000..316e741a8fa --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/query.h @@ -0,0 +1,71 @@ +#ifndef MARISA_QUERY_H_ +#define MARISA_QUERY_H_ + +#include "marisa/base.h" + +namespace marisa { + +class Query { + public: + Query() : ptr_(NULL), length_(0), id_(0) {} + Query(const Query &query) + : ptr_(query.ptr_), length_(query.length_), id_(query.id_) {} + + Query &operator=(const Query &query) { + ptr_ = query.ptr_; + length_ = query.length_; + id_ = query.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return ptr_[i]; + } + + void set_str(const char *str) { + MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR); + std::size_t length = 0; + while (str[length] != '\0') { + ++length; + } + ptr_ = str; + length_ = length; + } + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + ptr_ = ptr; + length_ = length; + } + void set_id(std::size_t id) { + id_ = id; + } + + const char *ptr() const { + return ptr_; + } + std::size_t length() const { + return length_; + } + std::size_t id() const { + return id_; + } + + void clear() { + Query().swap(*this); + } + void swap(Query &rhs) { + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(length_, rhs.length_); + marisa::swap(id_, rhs.id_); + } + + private: + const char *ptr_; + std::size_t length_; + std::size_t id_; +}; + +} // namespace marisa + +#endif // MARISA_QUERY_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/scoped-array.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/scoped-array.h new file mode 100644 index 00000000000..12b5b9e7f83 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/scoped-array.h @@ -0,0 +1,48 @@ +#ifndef MARISA_SCOPED_ARRAY_H_ +#define MARISA_SCOPED_ARRAY_H_ + +#include "marisa/base.h" + +namespace marisa { + +template <typename T> +class scoped_array { + public: + scoped_array() : array_(NULL) {} + explicit scoped_array(T *array) : array_(array) {} + + ~scoped_array() { + delete [] array_; + } + + void reset(T *array = NULL) { + MARISA_THROW_IF((array != NULL) && (array == array_), MARISA_RESET_ERROR); + scoped_array(array).swap(*this); + } + + T &operator[](std::size_t i) const { + MARISA_DEBUG_IF(array_ == NULL, MARISA_STATE_ERROR); + return array_[i]; + } + T *get() const { + return array_; + } + + void clear() { + scoped_array().swap(*this); + } + void swap(scoped_array &rhs) { + marisa::swap(array_, rhs.array_); + } + + private: + T *array_; + + // Disallows copy and assignment. + scoped_array(const scoped_array &); + scoped_array &operator=(const scoped_array &); +}; + +} // namespace marisa + +#endif // MARISA_SCOPED_ARRAY_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/scoped-ptr.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/scoped-ptr.h new file mode 100644 index 00000000000..63d7a3dcc92 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/scoped-ptr.h @@ -0,0 +1,52 @@ +#ifndef MARISA_SCOPED_PTR_H_ +#define MARISA_SCOPED_PTR_H_ + +#include "marisa/base.h" + +namespace marisa { + +template <typename T> +class scoped_ptr { + public: + scoped_ptr() : ptr_(NULL) {} + explicit scoped_ptr(T *ptr) : ptr_(ptr) {} + + ~scoped_ptr() { + delete ptr_; + } + + void reset(T *ptr = NULL) { + MARISA_THROW_IF((ptr != NULL) && (ptr == ptr_), MARISA_RESET_ERROR); + scoped_ptr(ptr).swap(*this); + } + + T &operator*() const { + MARISA_DEBUG_IF(ptr_ == NULL, MARISA_STATE_ERROR); + return *ptr_; + } + T *operator->() const { + MARISA_DEBUG_IF(ptr_ == NULL, MARISA_STATE_ERROR); + return ptr_; + } + T *get() const { + return ptr_; + } + + void clear() { + scoped_ptr().swap(*this); + } + void swap(scoped_ptr &rhs) { + marisa::swap(ptr_, rhs.ptr_); + } + + private: + T *ptr_; + + // Disallows copy and assignment. + scoped_ptr(const scoped_ptr &); + scoped_ptr &operator=(const scoped_ptr &); +}; + +} // namespace marisa + +#endif // MARISA_SCOPED_PTR_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/stdio.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/stdio.h new file mode 100644 index 00000000000..ee9a7b7795f --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/stdio.h @@ -0,0 +1,15 @@ +#ifndef MARISA_MYSTDIO_H_ +#define MARISA_MYSTDIO_H_ + +#include <cstdio> + +namespace marisa { + +class Trie; + +void fread(std::FILE *file, Trie *trie); +void fwrite(std::FILE *file, const Trie &trie); + +} // namespace marisa + +#endif // MARISA_MYSTDIO_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/trie.h b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/trie.h new file mode 100644 index 00000000000..30f3c686b26 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/include/marisa/trie.h @@ -0,0 +1,64 @@ +#ifndef MARISA_TRIE_H_ +#define MARISA_TRIE_H_ + +#include "marisa/keyset.h" +#include "marisa/agent.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class LoudsTrie; + +} // namespace trie +} // namespace grimoire + +class Trie { + friend class TrieIO; + + public: + Trie(); + ~Trie(); + + void build(Keyset &keyset, int config_flags = 0); + + void mmap(const char *filename); + void map(const void *ptr, std::size_t size); + + void load(const char *filename); + void read(int fd); + + void save(const char *filename) const; + void write(int fd) const; + + bool lookup(Agent &agent) const; + void reverse_lookup(Agent &agent) const; + bool common_prefix_search(Agent &agent) const; + bool predictive_search(Agent &agent) const; + + std::size_t num_tries() const; + std::size_t num_keys() const; + std::size_t num_nodes() const; + + TailMode tail_mode() const; + NodeOrder node_order() const; + + bool empty() const; + std::size_t size() const; + std::size_t total_size() const; + std::size_t io_size() const; + + void clear(); + void swap(Trie &rhs); + + private: + scoped_ptr<grimoire::trie::LoudsTrie> trie_; + + // Disallows copy and assignment. + Trie(const Trie &); + Trie &operator=(const Trie &); +}; + +} // namespace marisa + +#endif // MARISA_TRIE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/agent.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/agent.cc new file mode 100644 index 00000000000..7fa7cb1df91 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/agent.cc @@ -0,0 +1,51 @@ +#include <new> + +#include "marisa/agent.h" +#include "marisa/grimoire/trie.h" + +namespace marisa { + +Agent::Agent() : query_(), key_(), state_() {} + +Agent::~Agent() {} + +void Agent::set_query(const char *str) { + MARISA_THROW_IF(str == NULL, MARISA_NULL_ERROR); + if (state_.get() != NULL) { + state_->reset(); + } + query_.set_str(str); +} + +void Agent::set_query(const char *ptr, std::size_t length) { + MARISA_THROW_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + if (state_.get() != NULL) { + state_->reset(); + } + query_.set_str(ptr, length); +} + +void Agent::set_query(std::size_t key_id) { + if (state_.get() != NULL) { + state_->reset(); + } + query_.set_id(key_id); +} + +void Agent::init_state() { + MARISA_THROW_IF(state_.get() != NULL, MARISA_STATE_ERROR); + state_.reset(new (std::nothrow) grimoire::State); + MARISA_THROW_IF(state_.get() == NULL, MARISA_MEMORY_ERROR); +} + +void Agent::clear() { + Agent().swap(*this); +} + +void Agent::swap(Agent &rhs) { + query_.swap(rhs.query_); + key_.swap(rhs.key_); + state_.swap(rhs.state_); +} + +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/algorithm.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/algorithm.h new file mode 100644 index 00000000000..970c09f0eaf --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/algorithm.h @@ -0,0 +1,26 @@ +#ifndef MARISA_GRIMOIRE_ALGORITHM_H_ +#define MARISA_GRIMOIRE_ALGORITHM_H_ + +#include "marisa/grimoire/algorithm/sort.h" + +namespace marisa { +namespace grimoire { + +class Algorithm { + public: + Algorithm() {} + + template <typename Iterator> + std::size_t sort(Iterator begin, Iterator end) const { + return algorithm::sort(begin, end); + } + + private: + Algorithm(const Algorithm &); + Algorithm &operator=(const Algorithm &); +}; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_ALGORITHM_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/algorithm/sort.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/algorithm/sort.h new file mode 100644 index 00000000000..5f3b6526bda --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/algorithm/sort.h @@ -0,0 +1,196 @@ +#ifndef MARISA_GRIMOIRE_ALGORITHM_SORT_H_ +#define MARISA_GRIMOIRE_ALGORITHM_SORT_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace algorithm { +namespace details { + +enum { + MARISA_INSERTION_SORT_THRESHOLD = 10 +}; + +template <typename T> +int get_label(const T &unit, std::size_t depth) { + MARISA_DEBUG_IF(depth > unit.length(), MARISA_BOUND_ERROR); + + return (depth < unit.length()) ? (int)(UInt8)unit[depth] : -1; +} + +template <typename T> +int median(const T &a, const T &b, const T &c, std::size_t depth) { + const int x = get_label(a, depth); + const int y = get_label(b, depth); + const int z = get_label(c, depth); + if (x < y) { + if (y < z) { + return y; + } else if (x < z) { + return z; + } + return x; + } else if (x < z) { + return x; + } else if (y < z) { + return z; + } + return y; +} + +template <typename T> +int compare(const T &lhs, const T &rhs, std::size_t depth) { + for (std::size_t i = depth; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return 1; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] - (UInt8)rhs[i]; + } + } + if (lhs.length() == rhs.length()) { + return 0; + } + return (lhs.length() < rhs.length()) ? -1 : 1; +} + +template <typename Iterator> +std::size_t insertion_sort(Iterator l, Iterator r, std::size_t depth) { + MARISA_DEBUG_IF(l > r, MARISA_BOUND_ERROR); + + std::size_t count = 1; + for (Iterator i = l + 1; i < r; ++i) { + int result = 0; + for (Iterator j = i; j > l; --j) { + result = compare(*(j - 1), *j, depth); + if (result <= 0) { + break; + } + marisa::swap(*(j - 1), *j); + } + if (result != 0) { + ++count; + } + } + return count; +} + +template <typename Iterator> +std::size_t sort(Iterator l, Iterator r, std::size_t depth) { + MARISA_DEBUG_IF(l > r, MARISA_BOUND_ERROR); + + std::size_t count = 0; + while ((r - l) > MARISA_INSERTION_SORT_THRESHOLD) { + Iterator pl = l; + Iterator pr = r; + Iterator pivot_l = l; + Iterator pivot_r = r; + + const int pivot = median(*l, *(l + (r - l) / 2), *(r - 1), depth); + for ( ; ; ) { + while (pl < pr) { + const int label = get_label(*pl, depth); + if (label > pivot) { + break; + } else if (label == pivot) { + marisa::swap(*pl, *pivot_l); + ++pivot_l; + } + ++pl; + } + while (pl < pr) { + const int label = get_label(*--pr, depth); + if (label < pivot) { + break; + } else if (label == pivot) { + marisa::swap(*pr, *--pivot_r); + } + } + if (pl >= pr) { + break; + } + marisa::swap(*pl, *pr); + ++pl; + } + while (pivot_l > l) { + marisa::swap(*--pivot_l, *--pl); + } + while (pivot_r < r) { + marisa::swap(*pivot_r, *pr); + ++pivot_r; + ++pr; + } + + if (((pl - l) > (pr - pl)) || ((r - pr) > (pr - pl))) { + if ((pr - pl) == 1) { + ++count; + } else if ((pr - pl) > 1) { + if (pivot == -1) { + ++count; + } else { + count += sort(pl, pr, depth + 1); + } + } + + if ((pl - l) < (r - pr)) { + if ((pl - l) == 1) { + ++count; + } else if ((pl - l) > 1) { + count += sort(l, pl, depth); + } + l = pr; + } else { + if ((r - pr) == 1) { + ++count; + } else if ((r - pr) > 1) { + count += sort(pr, r, depth); + } + r = pl; + } + } else { + if ((pl - l) == 1) { + ++count; + } else if ((pl - l) > 1) { + count += sort(l, pl, depth); + } + + if ((r - pr) == 1) { + ++count; + } else if ((r - pr) > 1) { + count += sort(pr, r, depth); + } + + l = pl, r = pr; + if ((pr - pl) == 1) { + ++count; + } else if ((pr - pl) > 1) { + if (pivot == -1) { + l = r; + ++count; + } else { + ++depth; + } + } + } + } + + if ((r - l) > 1) { + count += insertion_sort(l, r, depth); + } + return count; +} + +} // namespace details + +template <typename Iterator> +std::size_t sort(Iterator begin, Iterator end) { + MARISA_DEBUG_IF(begin > end, MARISA_BOUND_ERROR); + return details::sort(begin, end, 0); +}; + +} // namespace algorithm +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_ALGORITHM_SORT_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/intrin.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/intrin.h new file mode 100644 index 00000000000..77b4e9980f5 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/intrin.h @@ -0,0 +1,138 @@ +#ifndef MARISA_GRIMOIRE_INTRIN_H_ +#define MARISA_GRIMOIRE_INTRIN_H_ + +#include "marisa/base.h" + +#if defined(__x86_64__) || defined(_M_X64) + #define MARISA_X64 +#elif defined(__i386__) || defined(_M_IX86) + #define MARISA_X86 +#else // defined(__i386__) || defined(_M_IX86) + #ifdef MARISA_USE_BMI2 + #undef MARISA_USE_BMI2 + #endif // MARISA_USE_BMI2 + #ifdef MARISA_USE_BMI + #undef MARISA_USE_BMI + #endif // MARISA_USE_BMI + #ifdef MARISA_USE_POPCNT + #undef MARISA_USE_POPCNT + #endif // MARISA_USE_POPCNT + #ifdef MARISA_USE_SSE4A + #undef MARISA_USE_SSE4A + #endif // MARISA_USE_SSE4A + #ifdef MARISA_USE_SSE4 + #undef MARISA_USE_SSE4 + #endif // MARISA_USE_SSE4 + #ifdef MARISA_USE_SSE4_2 + #undef MARISA_USE_SSE4_2 + #endif // MARISA_USE_SSE4_2 + #ifdef MARISA_USE_SSE4_1 + #undef MARISA_USE_SSE4_1 + #endif // MARISA_USE_SSE4_1 + #ifdef MARISA_USE_SSSE3 + #undef MARISA_USE_SSSE3 + #endif // MARISA_USE_SSSE3 + #ifdef MARISA_USE_SSE3 + #undef MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifdef MARISA_USE_SSE2 + #undef MARISA_USE_SSE2 + #endif // MARISA_USE_SSE2 +#endif // defined(__i386__) || defined(_M_IX86) + +#ifdef MARISA_USE_BMI2 + #ifndef MARISA_USE_BMI + #define MARISA_USE_BMI + #endif // MARISA_USE_BMI + #ifdef _MSC_VER + #include <immintrin.h> + #else // _MSC_VER + #include <x86intrin.h> + #endif // _MSC_VER +#endif // MARISA_USE_BMI2 + +#ifdef MARISA_USE_BMI + #ifndef MARISA_USE_SSE4 + #define MARISA_USE_SSE4 + #endif // MARISA_USE_SSE4 +#endif // MARISA_USE_BMI + +#ifdef MARISA_USE_SSE4A + #ifndef MARISA_USE_SSE3 + #define MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifndef MARISA_USE_POPCNT + #define MARISA_USE_POPCNT + #endif // MARISA_USE_POPCNT +#endif // MARISA_USE_SSE4A + +#ifdef MARISA_USE_SSE4 + #ifndef MARISA_USE_SSE4_2 + #define MARISA_USE_SSE4_2 + #endif // MARISA_USE_SSE4_2 +#endif // MARISA_USE_SSE4 + +#ifdef MARISA_USE_SSE4_2 + #ifndef MARISA_USE_SSE4_1 + #define MARISA_USE_SSE4_1 + #endif // MARISA_USE_SSE4_1 + #ifndef MARISA_USE_POPCNT + #define MARISA_USE_POPCNT + #endif // MARISA_USE_POPCNT +#endif // MARISA_USE_SSE4_2 + +#ifdef MARISA_USE_SSE4_1 + #ifndef MARISA_USE_SSSE3 + #define MARISA_USE_SSSE3 + #endif // MARISA_USE_SSSE3 +#endif // MARISA_USE_SSE4_1 + +#ifdef MARISA_USE_POPCNT + #ifndef MARISA_USE_SSE3 + #define MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifdef _MSC_VER + #include <intrin.h> + #else // _MSC_VER + #include <popcntintrin.h> + #endif // _MSC_VER +#endif // MARISA_USE_POPCNT + +#ifdef MARISA_USE_SSSE3 + #ifndef MARISA_USE_SSE3 + #define MARISA_USE_SSE3 + #endif // MARISA_USE_SSE3 + #ifdef MARISA_X64 + #define MARISA_X64_SSSE3 + #else // MARISA_X64 + #define MARISA_X86_SSSE3 + #endif // MAIRSA_X64 + #include <tmmintrin.h> +#endif // MARISA_USE_SSSE3 + +#ifdef MARISA_USE_SSE3 + #ifndef MARISA_USE_SSE2 + #define MARISA_USE_SSE2 + #endif // MARISA_USE_SSE2 +#endif // MARISA_USE_SSE3 + +#ifdef MARISA_USE_SSE2 + #ifdef MARISA_X64 + #define MARISA_X64_SSE2 + #else // MARISA_X64 + #define MARISA_X86_SSE2 + #endif // MAIRSA_X64 + #include <emmintrin.h> +#endif // MARISA_USE_SSE2 + +#ifdef _MSC_VER + #if MARISA_WORD_SIZE == 64 + #include <intrin.h> + #pragma intrinsic(_BitScanForward64) + #else // MARISA_WORD_SIZE == 64 + #include <intrin.h> + #pragma intrinsic(_BitScanForward) + #endif // MARISA_WORD_SIZE == 64 +#endif // _MSC_VER + +#endif // MARISA_GRIMOIRE_INTRIN_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io.h new file mode 100644 index 00000000000..43ff8c91cf2 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io.h @@ -0,0 +1,18 @@ +#ifndef MARISA_GRIMOIRE_IO_H_ +#define MARISA_GRIMOIRE_IO_H_ + +#include "marisa/grimoire/io/mapper.h" +#include "marisa/grimoire/io/reader.h" +#include "marisa/grimoire/io/writer.h" + +namespace marisa { +namespace grimoire { + +using io::Mapper; +using io::Reader; +using io::Writer; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/mapper.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/mapper.cc new file mode 100644 index 00000000000..6ae0ee37f39 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/mapper.cc @@ -0,0 +1,163 @@ +#if (defined _WIN32) || (defined _WIN64) + #include <sys/types.h> + #include <sys/stat.h> + #include <windows.h> +#else // (defined _WIN32) || (defined _WIN64) + #include <sys/mman.h> + #include <sys/stat.h> + #include <sys/types.h> + #include <fcntl.h> + #include <unistd.h> +#endif // (defined _WIN32) || (defined _WIN64) + +#include "marisa/grimoire/io/mapper.h" + +namespace marisa { +namespace grimoire { +namespace io { + +#if (defined _WIN32) || (defined _WIN64) +Mapper::Mapper() + : ptr_(NULL), origin_(NULL), avail_(0), size_(0), + file_(NULL), map_(NULL) {} +#else // (defined _WIN32) || (defined _WIN64) +Mapper::Mapper() + : ptr_(NULL), origin_(MAP_FAILED), avail_(0), size_(0), fd_(-1) {} +#endif // (defined _WIN32) || (defined _WIN64) + +#if (defined _WIN32) || (defined _WIN64) +Mapper::~Mapper() { + if (origin_ != NULL) { + ::UnmapViewOfFile(origin_); + } + + if (map_ != NULL) { + ::CloseHandle(map_); + } + + if (file_ != NULL) { + ::CloseHandle(file_); + } +} +#else // (defined _WIN32) || (defined _WIN64) +Mapper::~Mapper() { + if (origin_ != MAP_FAILED) { + ::munmap(origin_, size_); + } + + if (fd_ != -1) { + ::close(fd_); + } +} +#endif // (defined _WIN32) || (defined _WIN64) + +void Mapper::open(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + Mapper temp; + temp.open_(filename); + swap(temp); +} + +void Mapper::open(const void *ptr, std::size_t size) { + MARISA_THROW_IF((ptr == NULL) && (size != 0), MARISA_NULL_ERROR); + + Mapper temp; + temp.open_(ptr, size); + swap(temp); +} + +void Mapper::seek(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + MARISA_THROW_IF(size > avail_, MARISA_IO_ERROR); + + map_data(size); +} + +bool Mapper::is_open() const { + return ptr_ != NULL; +} + +void Mapper::clear() { + Mapper().swap(*this); +} + +void Mapper::swap(Mapper &rhs) { + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(avail_, rhs.avail_); + marisa::swap(origin_, rhs.origin_); + marisa::swap(size_, rhs.size_); +#if (defined _WIN32) || (defined _WIN64) + marisa::swap(file_, rhs.file_); + marisa::swap(map_, rhs.map_); +#else // (defined _WIN32) || (defined _WIN64) + marisa::swap(fd_, rhs.fd_); +#endif // (defined _WIN32) || (defined _WIN64) +} + +const void *Mapper::map_data(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + MARISA_THROW_IF(size > avail_, MARISA_IO_ERROR); + + const char * const data = static_cast<const char *>(ptr_); + ptr_ = data + size; + avail_ -= size; + return data; +} + +#if (defined _WIN32) || (defined _WIN64) + #ifdef __MSVCRT_VERSION__ + #if __MSVCRT_VERSION__ >= 0x0601 + #define MARISA_HAS_STAT64 + #endif // __MSVCRT_VERSION__ >= 0x0601 + #endif // __MSVCRT_VERSION__ +void Mapper::open_(const char *filename) { + #ifdef MARISA_HAS_STAT64 + struct __stat64 st; + MARISA_THROW_IF(::_stat64(filename, &st) != 0, MARISA_IO_ERROR); + #else // MARISA_HAS_STAT64 + struct _stat st; + MARISA_THROW_IF(::_stat(filename, &st) != 0, MARISA_IO_ERROR); + #endif // MARISA_HAS_STAT64 + MARISA_THROW_IF((UInt64)st.st_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)st.st_size; + + file_ = ::CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + MARISA_THROW_IF(file_ == INVALID_HANDLE_VALUE, MARISA_IO_ERROR); + + map_ = ::CreateFileMapping(file_, NULL, PAGE_READONLY, 0, 0, NULL); + MARISA_THROW_IF(map_ == NULL, MARISA_IO_ERROR); + + origin_ = ::MapViewOfFile(map_, FILE_MAP_READ, 0, 0, 0); + MARISA_THROW_IF(origin_ == NULL, MARISA_IO_ERROR); + + ptr_ = static_cast<const char *>(origin_); + avail_ = size_; +} +#else // (defined _WIN32) || (defined _WIN64) +void Mapper::open_(const char *filename) { + struct stat st; + MARISA_THROW_IF(::stat(filename, &st) != 0, MARISA_IO_ERROR); + MARISA_THROW_IF((UInt64)st.st_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)st.st_size; + + fd_ = ::open(filename, O_RDONLY); + MARISA_THROW_IF(fd_ == -1, MARISA_IO_ERROR); + + origin_ = ::mmap(NULL, size_, PROT_READ, MAP_SHARED, fd_, 0); + MARISA_THROW_IF(origin_ == MAP_FAILED, MARISA_IO_ERROR); + + ptr_ = static_cast<const char *>(origin_); + avail_ = size_; +} +#endif // (defined _WIN32) || (defined _WIN64) + +void Mapper::open_(const void *ptr, std::size_t size) { + ptr_ = ptr; + avail_ = size; +} + +} // namespace io +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/mapper.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/mapper.h new file mode 100644 index 00000000000..f701c9477a6 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/mapper.h @@ -0,0 +1,67 @@ +#ifndef MARISA_GRIMOIRE_IO_MAPPER_H_ +#define MARISA_GRIMOIRE_IO_MAPPER_H_ + +#include <cstdio> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace io { + +class Mapper { + public: + Mapper(); + ~Mapper(); + + void open(const char *filename); + void open(const void *ptr, std::size_t size); + + template <typename T> + void map(T *obj) { + MARISA_THROW_IF(obj == NULL, MARISA_NULL_ERROR); + *obj = *static_cast<const T *>(map_data(sizeof(T))); + } + + template <typename T> + void map(const T **objs, std::size_t num_objs) { + MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)), + MARISA_SIZE_ERROR); + *objs = static_cast<const T *>(map_data(sizeof(T) * num_objs)); + } + + void seek(std::size_t size); + + bool is_open() const; + + void clear(); + void swap(Mapper &rhs); + + private: + const void *ptr_; + void *origin_; + std::size_t avail_; + std::size_t size_; +#if (defined _WIN32) || (defined _WIN64) + void *file_; + void *map_; +#else // (defined _WIN32) || (defined _WIN64) + int fd_; +#endif // (defined _WIN32) || (defined _WIN64) + + void open_(const char *filename); + void open_(const void *ptr, std::size_t size); + + const void *map_data(std::size_t size); + + // Disallows copy and assignment. + Mapper(const Mapper &); + Mapper &operator=(const Mapper &); +}; + +} // namespace io +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_MAPPER_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/reader.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/reader.cc new file mode 100644 index 00000000000..b9c0809886a --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/reader.cc @@ -0,0 +1,147 @@ +#include <stdio.h> + +#ifdef _WIN32 + #include <io.h> +#else // _WIN32 + #include <unistd.h> +#endif // _WIN32 + +#include <limits> + +#include "marisa/grimoire/io/reader.h" + +namespace marisa { +namespace grimoire { +namespace io { + +Reader::Reader() + : file_(NULL), fd_(-1), stream_(NULL), needs_fclose_(false) {} + +Reader::~Reader() { + if (needs_fclose_) { + ::fclose(file_); + } +} + +void Reader::open(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + Reader temp; + temp.open_(filename); + swap(temp); +} + +void Reader::open(std::FILE *file) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + + Reader temp; + temp.open_(file); + swap(temp); +} + +void Reader::open(int fd) { + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + Reader temp; + temp.open_(fd); + swap(temp); +} + +void Reader::open(std::istream &stream) { + Reader temp; + temp.open_(stream); + swap(temp); +} + +void Reader::clear() { + Reader().swap(*this); +} + +void Reader::swap(Reader &rhs) { + marisa::swap(file_, rhs.file_); + marisa::swap(fd_, rhs.fd_); + marisa::swap(stream_, rhs.stream_); + marisa::swap(needs_fclose_, rhs.needs_fclose_); +} + +void Reader::seek(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (size <= 16) { + char buf[16]; + read_data(buf, size); + } else { + char buf[1024]; + while (size != 0) { + const std::size_t count = (size < sizeof(buf)) ? size : sizeof(buf); + read_data(buf, count); + size -= count; + } + } +} + +bool Reader::is_open() const { + return (file_ != NULL) || (fd_ != -1) || (stream_ != NULL); +} + +void Reader::open_(const char *filename) { + std::FILE *file = NULL; +#ifdef _MSC_VER + MARISA_THROW_IF(::fopen_s(&file, filename, "rb") != 0, MARISA_IO_ERROR); +#else // _MSC_VER + file = ::fopen(filename, "rb"); + MARISA_THROW_IF(file == NULL, MARISA_IO_ERROR); +#endif // _MSC_VER + file_ = file; + needs_fclose_ = true; +} + +void Reader::open_(std::FILE *file) { + file_ = file; +} + +void Reader::open_(int fd) { + fd_ = fd; +} + +void Reader::open_(std::istream &stream) { + stream_ = &stream; +} + +void Reader::read_data(void *buf, std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (fd_ != -1) { + while (size != 0) { +#ifdef _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits<int>::max(); + const unsigned int count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const int size_read = ::_read(fd_, buf, count); +#else // _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits< ::ssize_t>::max(); + const ::size_t count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const ::ssize_t size_read = ::read(fd_, buf, count); +#endif // _WIN32 + MARISA_THROW_IF(size_read <= 0, MARISA_IO_ERROR); + buf = static_cast<char *>(buf) + size_read; + size -= static_cast<std::size_t>(size_read); + } + } else if (file_ != NULL) { + MARISA_THROW_IF(::fread(buf, 1, size, file_) != size, MARISA_IO_ERROR); + } else if (stream_ != NULL) { + try { + MARISA_THROW_IF(!stream_->read(static_cast<char *>(buf), + static_cast<std::streamsize>(size)), MARISA_IO_ERROR); + } catch (const std::ios_base::failure &) { + MARISA_THROW(MARISA_IO_ERROR, "std::ios_base::failure"); + } + } +} + +} // namespace io +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/reader.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/reader.h new file mode 100644 index 00000000000..fa4ce362782 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/reader.h @@ -0,0 +1,66 @@ +#ifndef MARISA_GRIMOIRE_IO_READER_H_ +#define MARISA_GRIMOIRE_IO_READER_H_ + +#include <cstdio> +#include <iostream> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace io { + +class Reader { + public: + Reader(); + ~Reader(); + + void open(const char *filename); + void open(std::FILE *file); + void open(int fd); + void open(std::istream &stream); + + template <typename T> + void read(T *obj) { + MARISA_THROW_IF(obj == NULL, MARISA_NULL_ERROR); + read_data(obj, sizeof(T)); + } + + template <typename T> + void read(T *objs, std::size_t num_objs) { + MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)), + MARISA_SIZE_ERROR); + read_data(objs, sizeof(T) * num_objs); + } + + void seek(std::size_t size); + + bool is_open() const; + + void clear(); + void swap(Reader &rhs); + + private: + std::FILE *file_; + int fd_; + std::istream *stream_; + bool needs_fclose_; + + void open_(const char *filename); + void open_(std::FILE *file); + void open_(int fd); + void open_(std::istream &stream); + + void read_data(void *buf, std::size_t size); + + // Disallows copy and assignment. + Reader(const Reader &); + Reader &operator=(const Reader &); +}; + +} // namespace io +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_READER_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/writer.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/writer.cc new file mode 100644 index 00000000000..fb3d2d08ff9 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/writer.cc @@ -0,0 +1,148 @@ +#include <stdio.h> + +#ifdef _WIN32 + #include <io.h> +#else // _WIN32 + #include <unistd.h> +#endif // _WIN32 + +#include <limits> + +#include "marisa/grimoire/io/writer.h" + +namespace marisa { +namespace grimoire { +namespace io { + +Writer::Writer() + : file_(NULL), fd_(-1), stream_(NULL), needs_fclose_(false) {} + +Writer::~Writer() { + if (needs_fclose_) { + ::fclose(file_); + } +} + +void Writer::open(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + Writer temp; + temp.open_(filename); + swap(temp); +} + +void Writer::open(std::FILE *file) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + + Writer temp; + temp.open_(file); + swap(temp); +} + +void Writer::open(int fd) { + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + Writer temp; + temp.open_(fd); + swap(temp); +} + +void Writer::open(std::ostream &stream) { + Writer temp; + temp.open_(stream); + swap(temp); +} + +void Writer::clear() { + Writer().swap(*this); +} + +void Writer::swap(Writer &rhs) { + marisa::swap(file_, rhs.file_); + marisa::swap(fd_, rhs.fd_); + marisa::swap(stream_, rhs.stream_); + marisa::swap(needs_fclose_, rhs.needs_fclose_); +} + +void Writer::seek(std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (size <= 16) { + const char buf[16] = {}; + write_data(buf, size); + } else { + const char buf[1024] = {}; + do { + const std::size_t count = (size < sizeof(buf)) ? size : sizeof(buf); + write_data(buf, count); + size -= count; + } while (size != 0); + } +} + +bool Writer::is_open() const { + return (file_ != NULL) || (fd_ != -1) || (stream_ != NULL); +} + +void Writer::open_(const char *filename) { + std::FILE *file = NULL; +#ifdef _MSC_VER + MARISA_THROW_IF(::fopen_s(&file, filename, "wb") != 0, MARISA_IO_ERROR); +#else // _MSC_VER + file = ::fopen(filename, "wb"); + MARISA_THROW_IF(file == NULL, MARISA_IO_ERROR); +#endif // _MSC_VER + file_ = file; + needs_fclose_ = true; +} + +void Writer::open_(std::FILE *file) { + file_ = file; +} + +void Writer::open_(int fd) { + fd_ = fd; +} + +void Writer::open_(std::ostream &stream) { + stream_ = &stream; +} + +void Writer::write_data(const void *data, std::size_t size) { + MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR); + if (size == 0) { + return; + } else if (fd_ != -1) { + while (size != 0) { +#ifdef _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits<int>::max(); + const unsigned int count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const int size_written = ::_write(fd_, data, count); +#else // _WIN32 + static const std::size_t CHUNK_SIZE = + std::numeric_limits< ::ssize_t>::max(); + const ::size_t count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE; + const ::ssize_t size_written = ::write(fd_, data, count); +#endif // _WIN32 + MARISA_THROW_IF(size_written <= 0, MARISA_IO_ERROR); + data = static_cast<const char *>(data) + size_written; + size -= static_cast<std::size_t>(size_written); + } + } else if (file_ != NULL) { + MARISA_THROW_IF(::fwrite(data, 1, size, file_) != size, MARISA_IO_ERROR); + MARISA_THROW_IF(::fflush(file_) != 0, MARISA_IO_ERROR); + } else if (stream_ != NULL) { + try { + MARISA_THROW_IF(!stream_->write(static_cast<const char *>(data), + static_cast<std::streamsize>(size)), MARISA_IO_ERROR); + } catch (const std::ios_base::failure &) { + MARISA_THROW(MARISA_IO_ERROR, "std::ios_base::failure"); + } + } +} + +} // namespace io +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/writer.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/writer.h new file mode 100644 index 00000000000..d49761b0227 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/io/writer.h @@ -0,0 +1,65 @@ +#ifndef MARISA_GRIMOIRE_IO_WRITER_H_ +#define MARISA_GRIMOIRE_IO_WRITER_H_ + +#include <cstdio> +#include <iostream> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace io { + +class Writer { + public: + Writer(); + ~Writer(); + + void open(const char *filename); + void open(std::FILE *file); + void open(int fd); + void open(std::ostream &stream); + + template <typename T> + void write(const T &obj) { + write_data(&obj, sizeof(T)); + } + + template <typename T> + void write(const T *objs, std::size_t num_objs) { + MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)), + MARISA_SIZE_ERROR); + write_data(objs, sizeof(T) * num_objs); + } + + void seek(std::size_t size); + + bool is_open() const; + + void clear(); + void swap(Writer &rhs); + + private: + std::FILE *file_; + int fd_; + std::ostream *stream_; + bool needs_fclose_; + + void open_(const char *filename); + void open_(std::FILE *file); + void open_(int fd); + void open_(std::ostream &stream); + + void write_data(const void *data, std::size_t size); + + // Disallows copy and assignment. + Writer(const Writer &); + Writer &operator=(const Writer &); +}; + +} // namespace io +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_IO_WRITER_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie.h new file mode 100644 index 00000000000..73a0c2bf7aa --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie.h @@ -0,0 +1,16 @@ +#ifndef MARISA_GRIMOIRE_TRIE_H_ +#define MARISA_GRIMOIRE_TRIE_H_ + +#include "marisa/grimoire/trie/state.h" +#include "marisa/grimoire/trie/louds-trie.h" + +namespace marisa { +namespace grimoire { + +using trie::State; +using trie::LoudsTrie; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/cache.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/cache.h new file mode 100644 index 00000000000..19ce021c32d --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/cache.h @@ -0,0 +1,81 @@ +#ifndef MARISA_GRIMOIRE_TRIE_CACHE_H_ +#define MARISA_GRIMOIRE_TRIE_CACHE_H_ + +#include <cfloat> + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Cache { + public: + Cache() : parent_(0), child_(0), union_() { + union_.weight = FLT_MIN; + } + Cache(const Cache &cache) + : parent_(cache.parent_), child_(cache.child_), union_(cache.union_) {} + + Cache &operator=(const Cache &cache) { + parent_ = cache.parent_; + child_ = cache.child_; + union_ = cache.union_; + return *this; + } + + void set_parent(std::size_t parent) { + MARISA_DEBUG_IF(parent > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + parent_ = (UInt32)parent; + } + void set_child(std::size_t child) { + MARISA_DEBUG_IF(child > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + child_ = (UInt32)child; + } + void set_base(UInt8 base) { + union_.link = (union_.link & ~0xFFU) | base; + } + void set_extra(std::size_t extra) { + MARISA_DEBUG_IF(extra > (MARISA_UINT32_MAX >> 8), MARISA_SIZE_ERROR); + union_.link = (UInt32)((union_.link & 0xFFU) | (extra << 8)); + } + void set_weight(float weight) { + union_.weight = weight; + } + + std::size_t parent() const { + return parent_; + } + std::size_t child() const { + return child_; + } + UInt8 base() const { + return (UInt8)(union_.link & 0xFFU); + } + std::size_t extra() const { + return union_.link >> 8; + } + char label() const { + return (char)base(); + } + std::size_t link() const { + return union_.link; + } + float weight() const { + return union_.weight; + } + + private: + UInt32 parent_; + UInt32 child_; + union Union { + UInt32 link; + float weight; + } union_; +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_CACHE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/config.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/config.h new file mode 100644 index 00000000000..2f1e17a0643 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/config.h @@ -0,0 +1,155 @@ +#ifndef MARISA_GRIMOIRE_TRIE_CONFIG_H_ +#define MARISA_GRIMOIRE_TRIE_CONFIG_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Config { + public: + Config() + : num_tries_(MARISA_DEFAULT_NUM_TRIES), + cache_level_(MARISA_DEFAULT_CACHE), + tail_mode_(MARISA_DEFAULT_TAIL), + node_order_(MARISA_DEFAULT_ORDER) {} + + void parse(int config_flags) { + Config temp; + temp.parse_(config_flags); + swap(temp); + } + + int flags() const { + return (int)num_tries_ | tail_mode_ | node_order_; + } + + std::size_t num_tries() const { + return num_tries_; + } + CacheLevel cache_level() const { + return cache_level_; + } + TailMode tail_mode() const { + return tail_mode_; + } + NodeOrder node_order() const { + return node_order_; + } + + void clear() { + Config().swap(*this); + } + void swap(Config &rhs) { + marisa::swap(num_tries_, rhs.num_tries_); + marisa::swap(cache_level_, rhs.cache_level_); + marisa::swap(tail_mode_, rhs.tail_mode_); + marisa::swap(node_order_, rhs.node_order_); + } + + private: + std::size_t num_tries_; + CacheLevel cache_level_; + TailMode tail_mode_; + NodeOrder node_order_; + + void parse_(int config_flags) { + MARISA_THROW_IF((config_flags & ~MARISA_CONFIG_MASK) != 0, + MARISA_CODE_ERROR); + + parse_num_tries(config_flags); + parse_cache_level(config_flags); + parse_tail_mode(config_flags); + parse_node_order(config_flags); + } + + void parse_num_tries(int config_flags) { + const int num_tries = config_flags & MARISA_NUM_TRIES_MASK; + if (num_tries != 0) { + num_tries_ = static_cast<std::size_t>(num_tries); + } + } + + void parse_cache_level(int config_flags) { + switch (config_flags & MARISA_CACHE_LEVEL_MASK) { + case 0: { + cache_level_ = MARISA_DEFAULT_CACHE; + break; + } + case MARISA_HUGE_CACHE: { + cache_level_ = MARISA_HUGE_CACHE; + break; + } + case MARISA_LARGE_CACHE: { + cache_level_ = MARISA_LARGE_CACHE; + break; + } + case MARISA_NORMAL_CACHE: { + cache_level_ = MARISA_NORMAL_CACHE; + break; + } + case MARISA_SMALL_CACHE: { + cache_level_ = MARISA_SMALL_CACHE; + break; + } + case MARISA_TINY_CACHE: { + cache_level_ = MARISA_TINY_CACHE; + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined cache level"); + } + } + } + + void parse_tail_mode(int config_flags) { + switch (config_flags & MARISA_TAIL_MODE_MASK) { + case 0: { + tail_mode_ = MARISA_DEFAULT_TAIL; + break; + } + case MARISA_TEXT_TAIL: { + tail_mode_ = MARISA_TEXT_TAIL; + break; + } + case MARISA_BINARY_TAIL: { + tail_mode_ = MARISA_BINARY_TAIL; + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined tail mode"); + } + } + } + + void parse_node_order(int config_flags) { + switch (config_flags & MARISA_NODE_ORDER_MASK) { + case 0: { + node_order_ = MARISA_DEFAULT_ORDER; + break; + } + case MARISA_LABEL_ORDER: { + node_order_ = MARISA_LABEL_ORDER; + break; + } + case MARISA_WEIGHT_ORDER: { + node_order_ = MARISA_WEIGHT_ORDER; + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined node order"); + } + } + } + + // Disallows copy and assignment. + Config(const Config &); + Config &operator=(const Config &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_CONFIG_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/entry.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/entry.h new file mode 100644 index 00000000000..2c754a85335 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/entry.h @@ -0,0 +1,81 @@ +#ifndef MARISA_GRIMOIRE_TRIE_ENTRY_H_ +#define MARISA_GRIMOIRE_TRIE_ENTRY_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Entry { + public: + Entry() : ptr_(NULL), length_(0), id_(0) {} + Entry(const Entry &entry) + : ptr_(entry.ptr_), length_(entry.length_), id_(entry.id_) {} + + Entry &operator=(const Entry &entry) { + ptr_ = entry.ptr_; + length_ = entry.length_; + id_ = entry.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return *(ptr_ - i); + } + + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr + length - 1; + length_ = (UInt32)length; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + id_ = (UInt32)id; + } + + const char *ptr() const { + return ptr_ - length_ + 1; + } + std::size_t length() const { + return length_; + } + std::size_t id() const { + return id_; + } + + class StringComparer { + public: + bool operator()(const Entry &lhs, const Entry &rhs) const { + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return true; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] > (UInt8)rhs[i]; + } + } + return lhs.length() > rhs.length(); + } + }; + + class IDComparer { + public: + bool operator()(const Entry &lhs, const Entry &rhs) const { + return lhs.id_ < rhs.id_; + } + }; + + private: + const char *ptr_; + UInt32 length_; + UInt32 id_; +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_ENTRY_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/header.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/header.h new file mode 100644 index 00000000000..e13220c6a99 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/header.h @@ -0,0 +1,61 @@ +#ifndef MARISA_GRIMOIRE_TRIE_HEADER_H_ +#define MARISA_GRIMOIRE_TRIE_HEADER_H_ + +#include "marisa/grimoire/io.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Header { + public: + enum { + HEADER_SIZE = 16 + }; + + Header() {} + + void map(Mapper &mapper) { + const char *ptr; + mapper.map(&ptr, HEADER_SIZE); + MARISA_THROW_IF(!test_header(ptr), MARISA_FORMAT_ERROR); + } + void read(Reader &reader) { + char buf[HEADER_SIZE]; + reader.read(buf, HEADER_SIZE); + MARISA_THROW_IF(!test_header(buf), MARISA_FORMAT_ERROR); + } + void write(Writer &writer) const { + writer.write(get_header(), HEADER_SIZE); + } + + std::size_t io_size() const { + return HEADER_SIZE; + } + + private: + + static const char *get_header() { + static const char buf[HEADER_SIZE] = "We love Marisa."; + return buf; + } + + static bool test_header(const char *ptr) { + for (std::size_t i = 0; i < HEADER_SIZE; ++i) { + if (ptr[i] != get_header()[i]) { + return false; + } + } + return true; + } + + // Disallows copy and assignment. + Header(const Header &); + Header &operator=(const Header &); +}; + +} // namespace trie +} // namespace marisa +} // namespace grimoire + +#endif // MARISA_GRIMOIRE_TRIE_HEADER_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/history.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/history.h new file mode 100644 index 00000000000..84d10df1f19 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/history.h @@ -0,0 +1,65 @@ +#ifndef MARISA_GRIMOIRE_TRIE_STATE_HISTORY_H_ +#define MARISA_GRIMOIRE_TRIE_STATE_HISTORY_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class History { + public: + History() + : node_id_(0), louds_pos_(0), key_pos_(0), + link_id_(MARISA_INVALID_LINK_ID), key_id_(MARISA_INVALID_KEY_ID) {} + + void set_node_id(std::size_t node_id) { + MARISA_DEBUG_IF(node_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + node_id_ = (UInt32)node_id; + } + void set_louds_pos(std::size_t louds_pos) { + MARISA_DEBUG_IF(louds_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + louds_pos_ = (UInt32)louds_pos; + } + void set_key_pos(std::size_t key_pos) { + MARISA_DEBUG_IF(key_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_pos_ = (UInt32)key_pos; + } + void set_link_id(std::size_t link_id) { + MARISA_DEBUG_IF(link_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + link_id_ = (UInt32)link_id; + } + void set_key_id(std::size_t key_id) { + MARISA_DEBUG_IF(key_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_id_ = (UInt32)key_id; + } + + std::size_t node_id() const { + return node_id_; + } + std::size_t louds_pos() const { + return louds_pos_; + } + std::size_t key_pos() const { + return key_pos_; + } + std::size_t link_id() const { + return link_id_; + } + std::size_t key_id() const { + return key_id_; + } + + private: + UInt32 node_id_; + UInt32 louds_pos_; + UInt32 key_pos_; + UInt32 link_id_; + UInt32 key_id_; +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_STATE_HISTORY_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/key.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/key.h new file mode 100644 index 00000000000..8555cc72d2c --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/key.h @@ -0,0 +1,226 @@ +#ifndef MARISA_GRIMOIRE_TRIE_KEY_H_ +#define MARISA_GRIMOIRE_TRIE_KEY_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Key { + public: + Key() : ptr_(NULL), length_(0), union_(), id_(0) { + union_.terminal = 0; + } + Key(const Key &entry) + : ptr_(entry.ptr_), length_(entry.length_), + union_(entry.union_), id_(entry.id_) {} + + Key &operator=(const Key &entry) { + ptr_ = entry.ptr_; + length_ = entry.length_; + union_ = entry.union_; + id_ = entry.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return ptr_[i]; + } + + void substr(std::size_t pos, std::size_t length) { + MARISA_DEBUG_IF(pos > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(length > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(pos > (length_ - length), MARISA_BOUND_ERROR); + ptr_ += pos; + length_ = (UInt32)length; + } + + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr; + length_ = (UInt32)length; + } + void set_weight(float weight) { + union_.weight = weight; + } + void set_terminal(std::size_t terminal) { + MARISA_DEBUG_IF(terminal > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + union_.terminal = (UInt32)terminal; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + id_ = (UInt32)id; + } + + const char *ptr() const { + return ptr_; + } + std::size_t length() const { + return length_; + } + float weight() const { + return union_.weight; + } + std::size_t terminal() const { + return union_.terminal; + } + std::size_t id() const { + return id_; + } + + private: + const char *ptr_; + UInt32 length_; + union Union { + float weight; + UInt32 terminal; + } union_; + UInt32 id_; +}; + +inline bool operator==(const Key &lhs, const Key &rhs) { + if (lhs.length() != rhs.length()) { + return false; + } + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + +inline bool operator!=(const Key &lhs, const Key &rhs) { + return !(lhs == rhs); +} + +inline bool operator<(const Key &lhs, const Key &rhs) { + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return false; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] < (UInt8)rhs[i]; + } + } + return lhs.length() < rhs.length(); +} + +inline bool operator>(const Key &lhs, const Key &rhs) { + return rhs < lhs; +} + +class ReverseKey { + public: + ReverseKey() : ptr_(NULL), length_(0), union_(), id_(0) { + union_.terminal = 0; + } + ReverseKey(const ReverseKey &entry) + : ptr_(entry.ptr_), length_(entry.length_), + union_(entry.union_), id_(entry.id_) {} + + ReverseKey &operator=(const ReverseKey &entry) { + ptr_ = entry.ptr_; + length_ = entry.length_; + union_ = entry.union_; + id_ = entry.id_; + return *this; + } + + char operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR); + return *(ptr_ - i - 1); + } + + void substr(std::size_t pos, std::size_t length) { + MARISA_DEBUG_IF(pos > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(length > length_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(pos > (length_ - length), MARISA_BOUND_ERROR); + ptr_ -= pos; + length_ = (UInt32)length; + } + + void set_str(const char *ptr, std::size_t length) { + MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + ptr_ = ptr + length; + length_ = (UInt32)length; + } + void set_weight(float weight) { + union_.weight = weight; + } + void set_terminal(std::size_t terminal) { + MARISA_DEBUG_IF(terminal > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + union_.terminal = (UInt32)terminal; + } + void set_id(std::size_t id) { + MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + id_ = (UInt32)id; + } + + const char *ptr() const { + return ptr_ - length_; + } + std::size_t length() const { + return length_; + } + float weight() const { + return union_.weight; + } + std::size_t terminal() const { + return union_.terminal; + } + std::size_t id() const { + return id_; + } + + private: + const char *ptr_; + UInt32 length_; + union Union { + float weight; + UInt32 terminal; + } union_; + UInt32 id_; +}; + +inline bool operator==(const ReverseKey &lhs, const ReverseKey &rhs) { + if (lhs.length() != rhs.length()) { + return false; + } + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + +inline bool operator!=(const ReverseKey &lhs, const ReverseKey &rhs) { + return !(lhs == rhs); +} + +inline bool operator<(const ReverseKey &lhs, const ReverseKey &rhs) { + for (std::size_t i = 0; i < lhs.length(); ++i) { + if (i == rhs.length()) { + return false; + } + if (lhs[i] != rhs[i]) { + return (UInt8)lhs[i] < (UInt8)rhs[i]; + } + } + return lhs.length() < rhs.length(); +} + +inline bool operator>(const ReverseKey &lhs, const ReverseKey &rhs) { + return rhs < lhs; +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_KEY_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc new file mode 100644 index 00000000000..df191f5191e --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc @@ -0,0 +1,878 @@ +#include <algorithm> +#include <functional> +#include <queue> + +#include "marisa/grimoire/algorithm.h" +#include "marisa/grimoire/trie/header.h" +#include "marisa/grimoire/trie/range.h" +#include "marisa/grimoire/trie/state.h" +#include "marisa/grimoire/trie/louds-trie.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +LoudsTrie::LoudsTrie() + : louds_(), terminal_flags_(), link_flags_(), bases_(), extras_(), + tail_(), next_trie_(), cache_(), cache_mask_(0), num_l1_nodes_(0), + config_(), mapper_() {} + +LoudsTrie::~LoudsTrie() {} + +void LoudsTrie::build(Keyset &keyset, int flags) { + Config config; + config.parse(flags); + + LoudsTrie temp; + temp.build_(keyset, config); + swap(temp); +} + +void LoudsTrie::map(Mapper &mapper) { + Header().map(mapper); + + LoudsTrie temp; + temp.map_(mapper); + temp.mapper_.swap(mapper); + swap(temp); +} + +void LoudsTrie::read(Reader &reader) { + Header().read(reader); + + LoudsTrie temp; + temp.read_(reader); + swap(temp); +} + +void LoudsTrie::write(Writer &writer) const { + Header().write(writer); + + write_(writer); +} + +bool LoudsTrie::lookup(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + + State &state = agent.state(); + state.lookup_init(); + while (state.query_pos() < agent.query().length()) { + if (!find_child(agent)) { + return false; + } + } + if (!terminal_flags_[state.node_id()]) { + return false; + } + agent.set_key(agent.query().ptr(), agent.query().length()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; +} + +void LoudsTrie::reverse_lookup(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + MARISA_THROW_IF(agent.query().id() >= size(), MARISA_BOUND_ERROR); + + State &state = agent.state(); + state.reverse_lookup_init(); + + state.set_node_id(terminal_flags_.select1(agent.query().id())); + if (state.node_id() == 0) { + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(agent.query().id()); + return; + } + for ( ; ; ) { + if (link_flags_[state.node_id()]) { + const std::size_t prev_key_pos = state.key_buf().size(); + restore(agent, get_link(state.node_id())); + std::reverse(state.key_buf().begin() + prev_key_pos, + state.key_buf().end()); + } else { + state.key_buf().push_back((char)bases_[state.node_id()]); + } + + if (state.node_id() <= num_l1_nodes_) { + std::reverse(state.key_buf().begin(), state.key_buf().end()); + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(agent.query().id()); + return; + } + state.set_node_id(louds_.select1(state.node_id()) - state.node_id() - 1); + } +} + +bool LoudsTrie::common_prefix_search(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (state.status_code() == MARISA_END_OF_COMMON_PREFIX_SEARCH) { + return false; + } + + if (state.status_code() != MARISA_READY_TO_COMMON_PREFIX_SEARCH) { + state.common_prefix_search_init(); + if (terminal_flags_[state.node_id()]) { + agent.set_key(agent.query().ptr(), state.query_pos()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; + } + } + + while (state.query_pos() < agent.query().length()) { + if (!find_child(agent)) { + state.set_status_code(MARISA_END_OF_COMMON_PREFIX_SEARCH); + return false; + } else if (terminal_flags_[state.node_id()]) { + agent.set_key(agent.query().ptr(), state.query_pos()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; + } + } + state.set_status_code(MARISA_END_OF_COMMON_PREFIX_SEARCH); + return false; +} + +bool LoudsTrie::predictive_search(Agent &agent) const { + MARISA_DEBUG_IF(!agent.has_state(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (state.status_code() == MARISA_END_OF_PREDICTIVE_SEARCH) { + return false; + } + + if (state.status_code() != MARISA_READY_TO_PREDICTIVE_SEARCH) { + state.predictive_search_init(); + while (state.query_pos() < agent.query().length()) { + if (!predictive_find_child(agent)) { + state.set_status_code(MARISA_END_OF_PREDICTIVE_SEARCH); + return false; + } + } + + History history; + history.set_node_id(state.node_id()); + history.set_key_pos(state.key_buf().size()); + state.history().push_back(history); + state.set_history_pos(1); + + if (terminal_flags_[state.node_id()]) { + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(terminal_flags_.rank1(state.node_id())); + return true; + } + } + + for ( ; ; ) { + if (state.history_pos() == state.history().size()) { + const History ¤t = state.history().back(); + History next; + next.set_louds_pos(louds_.select0(current.node_id()) + 1); + next.set_node_id(next.louds_pos() - current.node_id() - 1); + state.history().push_back(next); + } + + History &next = state.history()[state.history_pos()]; + const bool link_flag = louds_[next.louds_pos()]; + next.set_louds_pos(next.louds_pos() + 1); + if (link_flag) { + state.set_history_pos(state.history_pos() + 1); + if (link_flags_[next.node_id()]) { + next.set_link_id(update_link_id(next.link_id(), next.node_id())); + restore(agent, get_link(next.node_id(), next.link_id())); + } else { + state.key_buf().push_back((char)bases_[next.node_id()]); + } + next.set_key_pos(state.key_buf().size()); + + if (terminal_flags_[next.node_id()]) { + if (next.key_id() == MARISA_INVALID_KEY_ID) { + next.set_key_id(terminal_flags_.rank1(next.node_id())); + } else { + next.set_key_id(next.key_id() + 1); + } + agent.set_key(state.key_buf().begin(), state.key_buf().size()); + agent.set_key(next.key_id()); + return true; + } + } else if (state.history_pos() != 1) { + History ¤t = state.history()[state.history_pos() - 1]; + current.set_node_id(current.node_id() + 1); + const History &prev = + state.history()[state.history_pos() - 2]; + state.key_buf().resize(prev.key_pos()); + state.set_history_pos(state.history_pos() - 1); + } else { + state.set_status_code(MARISA_END_OF_PREDICTIVE_SEARCH); + return false; + } + } +} + +std::size_t LoudsTrie::total_size() const { + return louds_.total_size() + terminal_flags_.total_size() + + link_flags_.total_size() + bases_.total_size() + + extras_.total_size() + tail_.total_size() + + ((next_trie_.get() != NULL) ? next_trie_->total_size() : 0) + + cache_.total_size(); +} + +std::size_t LoudsTrie::io_size() const { + return Header().io_size() + louds_.io_size() + + terminal_flags_.io_size() + link_flags_.io_size() + + bases_.io_size() + extras_.io_size() + tail_.io_size() + + ((next_trie_.get() != NULL) ? + (next_trie_->io_size() - Header().io_size()) : 0) + + cache_.io_size() + (sizeof(UInt32) * 2); +} + +void LoudsTrie::clear() { + LoudsTrie().swap(*this); +} + +void LoudsTrie::swap(LoudsTrie &rhs) { + louds_.swap(rhs.louds_); + terminal_flags_.swap(rhs.terminal_flags_); + link_flags_.swap(rhs.link_flags_); + bases_.swap(rhs.bases_); + extras_.swap(rhs.extras_); + tail_.swap(rhs.tail_); + next_trie_.swap(rhs.next_trie_); + cache_.swap(rhs.cache_); + marisa::swap(cache_mask_, rhs.cache_mask_); + marisa::swap(num_l1_nodes_, rhs.num_l1_nodes_); + config_.swap(rhs.config_); + mapper_.swap(rhs.mapper_); +} + +void LoudsTrie::build_(Keyset &keyset, const Config &config) { + Vector<Key> keys; + keys.resize(keyset.size()); + for (std::size_t i = 0; i < keyset.size(); ++i) { + keys[i].set_str(keyset[i].ptr(), keyset[i].length()); + keys[i].set_weight(keyset[i].weight()); + } + + Vector<UInt32> terminals; + build_trie(keys, &terminals, config, 1); + + typedef std::pair<UInt32, UInt32> TerminalIdPair; + + Vector<TerminalIdPair> pairs; + pairs.resize(terminals.size()); + for (std::size_t i = 0; i < pairs.size(); ++i) { + pairs[i].first = terminals[i]; + pairs[i].second = (UInt32)i; + } + terminals.clear(); + std::sort(pairs.begin(), pairs.end()); + + std::size_t node_id = 0; + for (std::size_t i = 0; i < pairs.size(); ++i) { + while (node_id < pairs[i].first) { + terminal_flags_.push_back(false); + ++node_id; + } + if (node_id == pairs[i].first) { + terminal_flags_.push_back(true); + ++node_id; + } + } + while (node_id < bases_.size()) { + terminal_flags_.push_back(false); + ++node_id; + } + terminal_flags_.push_back(false); + terminal_flags_.build(false, true); + + for (std::size_t i = 0; i < keyset.size(); ++i) { + keyset[pairs[i].second].set_id(terminal_flags_.rank1(pairs[i].first)); + } +} + +template <typename T> +void LoudsTrie::build_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id) { + build_current_trie(keys, terminals, config, trie_id); + + Vector<UInt32> next_terminals; + if (!keys.empty()) { + build_next_trie(keys, &next_terminals, config, trie_id); + } + + if (next_trie_.get() != NULL) { + config_.parse(static_cast<int>((next_trie_->num_tries() + 1)) | + next_trie_->tail_mode() | next_trie_->node_order()); + } else { + config_.parse(1 | tail_.mode() | config.node_order() | + config.cache_level()); + } + + link_flags_.build(false, false); + std::size_t node_id = 0; + for (std::size_t i = 0; i < next_terminals.size(); ++i) { + while (!link_flags_[node_id]) { + ++node_id; + } + bases_[node_id] = (UInt8)(next_terminals[i] % 256); + next_terminals[i] /= 256; + ++node_id; + } + extras_.build(next_terminals); + fill_cache(); +} + +template <typename T> +void LoudsTrie::build_current_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, + std::size_t trie_id) try { + for (std::size_t i = 0; i < keys.size(); ++i) { + keys[i].set_id(i); + } + const std::size_t num_keys = Algorithm().sort(keys.begin(), keys.end()); + reserve_cache(config, trie_id, num_keys); + + louds_.push_back(true); + louds_.push_back(false); + bases_.push_back('\0'); + link_flags_.push_back(false); + + Vector<T> next_keys; + std::queue<Range> queue; + Vector<WeightedRange> w_ranges; + + queue.push(make_range(0, keys.size(), 0)); + while (!queue.empty()) { + const std::size_t node_id = link_flags_.size() - queue.size(); + + Range range = queue.front(); + queue.pop(); + + while ((range.begin() < range.end()) && + (keys[range.begin()].length() == range.key_pos())) { + keys[range.begin()].set_terminal(node_id); + range.set_begin(range.begin() + 1); + } + + if (range.begin() == range.end()) { + louds_.push_back(false); + continue; + } + + w_ranges.clear(); + double weight = keys[range.begin()].weight(); + for (std::size_t i = range.begin() + 1; i < range.end(); ++i) { + if (keys[i - 1][range.key_pos()] != keys[i][range.key_pos()]) { + w_ranges.push_back(make_weighted_range( + range.begin(), i, range.key_pos(), (float)weight)); + range.set_begin(i); + weight = 0.0; + } + weight += keys[i].weight(); + } + w_ranges.push_back(make_weighted_range( + range.begin(), range.end(), range.key_pos(), (float)weight)); + if (config.node_order() == MARISA_WEIGHT_ORDER) { + std::stable_sort(w_ranges.begin(), w_ranges.end(), + std::greater<WeightedRange>()); + } + + if (node_id == 0) { + num_l1_nodes_ = w_ranges.size(); + } + + for (std::size_t i = 0; i < w_ranges.size(); ++i) { + WeightedRange &w_range = w_ranges[i]; + std::size_t key_pos = w_range.key_pos() + 1; + while (key_pos < keys[w_range.begin()].length()) { + std::size_t j; + for (j = w_range.begin() + 1; j < w_range.end(); ++j) { + if (keys[j - 1][key_pos] != keys[j][key_pos]) { + break; + } + } + if (j < w_range.end()) { + break; + } + ++key_pos; + } + cache<T>(node_id, bases_.size(), w_range.weight(), + keys[w_range.begin()][w_range.key_pos()]); + + if (key_pos == w_range.key_pos() + 1) { + bases_.push_back(static_cast<unsigned char>( + keys[w_range.begin()][w_range.key_pos()])); + link_flags_.push_back(false); + } else { + bases_.push_back('\0'); + link_flags_.push_back(true); + T next_key; + next_key.set_str(keys[w_range.begin()].ptr(), + keys[w_range.begin()].length()); + next_key.substr(w_range.key_pos(), key_pos - w_range.key_pos()); + next_key.set_weight(w_range.weight()); + next_keys.push_back(next_key); + } + w_range.set_key_pos(key_pos); + queue.push(w_range.range()); + louds_.push_back(true); + } + louds_.push_back(false); + } + + louds_.push_back(false); + louds_.build(trie_id == 1, true); + bases_.shrink(); + + build_terminals(keys, terminals); + keys.swap(next_keys); +} catch (const std::bad_alloc &) { + MARISA_THROW(MARISA_MEMORY_ERROR, "std::bad_alloc"); +} + +template <> +void LoudsTrie::build_next_trie(Vector<Key> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id) { + if (trie_id == config.num_tries()) { + Vector<Entry> entries; + entries.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + entries[i].set_str(keys[i].ptr(), keys[i].length()); + } + tail_.build(entries, terminals, config.tail_mode()); + return; + } + Vector<ReverseKey> reverse_keys; + reverse_keys.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + reverse_keys[i].set_str(keys[i].ptr(), keys[i].length()); + reverse_keys[i].set_weight(keys[i].weight()); + } + keys.clear(); + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->build_trie(reverse_keys, terminals, config, trie_id + 1); +} + +template <> +void LoudsTrie::build_next_trie(Vector<ReverseKey> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id) { + if (trie_id == config.num_tries()) { + Vector<Entry> entries; + entries.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + entries[i].set_str(keys[i].ptr(), keys[i].length()); + } + tail_.build(entries, terminals, config.tail_mode()); + return; + } + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->build_trie(keys, terminals, config, trie_id + 1); +} + +template <typename T> +void LoudsTrie::build_terminals(const Vector<T> &keys, + Vector<UInt32> *terminals) const { + Vector<UInt32> temp; + temp.resize(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + temp[keys[i].id()] = (UInt32)keys[i].terminal(); + } + terminals->swap(temp); +} + +template <> +void LoudsTrie::cache<Key>(std::size_t parent, std::size_t child, + float weight, char label) { + MARISA_DEBUG_IF(parent >= child, MARISA_RANGE_ERROR); + + const std::size_t cache_id = get_cache_id(parent, label); + if (weight > cache_[cache_id].weight()) { + cache_[cache_id].set_parent(parent); + cache_[cache_id].set_child(child); + cache_[cache_id].set_weight(weight); + } +} + +void LoudsTrie::reserve_cache(const Config &config, std::size_t trie_id, + std::size_t num_keys) { + std::size_t cache_size = (trie_id == 1) ? 256 : 1; + while (cache_size < (num_keys / config.cache_level())) { + cache_size *= 2; + } + cache_.resize(cache_size); + cache_mask_ = cache_size - 1; +} + +template <> +void LoudsTrie::cache<ReverseKey>(std::size_t parent, std::size_t child, + float weight, char) { + MARISA_DEBUG_IF(parent >= child, MARISA_RANGE_ERROR); + + const std::size_t cache_id = get_cache_id(child); + if (weight > cache_[cache_id].weight()) { + cache_[cache_id].set_parent(parent); + cache_[cache_id].set_child(child); + cache_[cache_id].set_weight(weight); + } +} + +void LoudsTrie::fill_cache() { + for (std::size_t i = 0; i < cache_.size(); ++i) { + const std::size_t node_id = cache_[i].child(); + if (node_id != 0) { + cache_[i].set_base(bases_[node_id]); + cache_[i].set_extra(!link_flags_[node_id] ? + MARISA_INVALID_EXTRA : extras_[link_flags_.rank1(node_id)]); + } else { + cache_[i].set_parent(MARISA_UINT32_MAX); + cache_[i].set_child(MARISA_UINT32_MAX); + } + } +} + +void LoudsTrie::map_(Mapper &mapper) { + louds_.map(mapper); + terminal_flags_.map(mapper); + link_flags_.map(mapper); + bases_.map(mapper); + extras_.map(mapper); + tail_.map(mapper); + if ((link_flags_.num_1s() != 0) && tail_.empty()) { + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->map_(mapper); + } + cache_.map(mapper); + cache_mask_ = cache_.size() - 1; + { + UInt32 temp_num_l1_nodes; + mapper.map(&temp_num_l1_nodes); + num_l1_nodes_ = temp_num_l1_nodes; + } + { + UInt32 temp_config_flags; + mapper.map(&temp_config_flags); + config_.parse((int)temp_config_flags); + } +} + +void LoudsTrie::read_(Reader &reader) { + louds_.read(reader); + terminal_flags_.read(reader); + link_flags_.read(reader); + bases_.read(reader); + extras_.read(reader); + tail_.read(reader); + if ((link_flags_.num_1s() != 0) && tail_.empty()) { + next_trie_.reset(new (std::nothrow) LoudsTrie); + MARISA_THROW_IF(next_trie_.get() == NULL, MARISA_MEMORY_ERROR); + next_trie_->read_(reader); + } + cache_.read(reader); + cache_mask_ = cache_.size() - 1; + { + UInt32 temp_num_l1_nodes; + reader.read(&temp_num_l1_nodes); + num_l1_nodes_ = temp_num_l1_nodes; + } + { + UInt32 temp_config_flags; + reader.read(&temp_config_flags); + config_.parse((int)temp_config_flags); + } +} + +void LoudsTrie::write_(Writer &writer) const { + louds_.write(writer); + terminal_flags_.write(writer); + link_flags_.write(writer); + bases_.write(writer); + extras_.write(writer); + tail_.write(writer); + if (next_trie_.get() != NULL) { + next_trie_->write_(writer); + } + cache_.write(writer); + writer.write((UInt32)num_l1_nodes_); + writer.write((UInt32)config_.flags()); +} + +bool LoudsTrie::find_child(Agent &agent) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + + State &state = agent.state(); + const std::size_t cache_id = get_cache_id(state.node_id(), + agent.query()[state.query_pos()]); + if (state.node_id() == cache_[cache_id].parent()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!match(agent, cache_[cache_id].link())) { + return false; + } + } else { + state.set_query_pos(state.query_pos() + 1); + } + state.set_node_id(cache_[cache_id].child()); + return true; + } + + std::size_t louds_pos = louds_.select0(state.node_id()) + 1; + if (!louds_[louds_pos]) { + return false; + } + state.set_node_id(louds_pos - state.node_id() - 1); + std::size_t link_id = MARISA_INVALID_LINK_ID; + do { + if (link_flags_[state.node_id()]) { + link_id = update_link_id(link_id, state.node_id()); + const std::size_t prev_query_pos = state.query_pos(); + if (match(agent, get_link(state.node_id(), link_id))) { + return true; + } else if (state.query_pos() != prev_query_pos) { + return false; + } + } else if (bases_[state.node_id()] == + (UInt8)agent.query()[state.query_pos()]) { + state.set_query_pos(state.query_pos() + 1); + return true; + } + state.set_node_id(state.node_id() + 1); + ++louds_pos; + } while (louds_[louds_pos]); + return false; +} + +bool LoudsTrie::predictive_find_child(Agent &agent) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + + State &state = agent.state(); + const std::size_t cache_id = get_cache_id(state.node_id(), + agent.query()[state.query_pos()]); + if (state.node_id() == cache_[cache_id].parent()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!prefix_match(agent, cache_[cache_id].link())) { + return false; + } + } else { + state.key_buf().push_back(cache_[cache_id].label()); + state.set_query_pos(state.query_pos() + 1); + } + state.set_node_id(cache_[cache_id].child()); + return true; + } + + std::size_t louds_pos = louds_.select0(state.node_id()) + 1; + if (!louds_[louds_pos]) { + return false; + } + state.set_node_id(louds_pos - state.node_id() - 1); + std::size_t link_id = MARISA_INVALID_LINK_ID; + do { + if (link_flags_[state.node_id()]) { + link_id = update_link_id(link_id, state.node_id()); + const std::size_t prev_query_pos = state.query_pos(); + if (prefix_match(agent, get_link(state.node_id(), link_id))) { + return true; + } else if (state.query_pos() != prev_query_pos) { + return false; + } + } else if (bases_[state.node_id()] == + (UInt8)agent.query()[state.query_pos()]) { + state.key_buf().push_back((char)bases_[state.node_id()]); + state.set_query_pos(state.query_pos() + 1); + return true; + } + state.set_node_id(state.node_id() + 1); + ++louds_pos; + } while (louds_[louds_pos]); + return false; +} + +void LoudsTrie::restore(Agent &agent, std::size_t link) const { + if (next_trie_.get() != NULL) { + next_trie_->restore_(agent, link); + } else { + tail_.restore(agent, link); + } +} + +bool LoudsTrie::match(Agent &agent, std::size_t link) const { + if (next_trie_.get() != NULL) { + return next_trie_->match_(agent, link); + } else { + return tail_.match(agent, link); + } +} + +bool LoudsTrie::prefix_match(Agent &agent, std::size_t link) const { + if (next_trie_.get() != NULL) { + return next_trie_->prefix_match_(agent, link); + } else { + return tail_.prefix_match(agent, link); + } +} + +void LoudsTrie::restore_(Agent &agent, std::size_t node_id) const { + MARISA_DEBUG_IF(node_id == 0, MARISA_RANGE_ERROR); + + State &state = agent.state(); + for ( ; ; ) { + const std::size_t cache_id = get_cache_id(node_id); + if (node_id == cache_[cache_id].child()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + restore(agent, cache_[cache_id].link()); + } else { + state.key_buf().push_back(cache_[cache_id].label()); + } + + node_id = cache_[cache_id].parent(); + if (node_id == 0) { + return; + } + continue; + } + + if (link_flags_[node_id]) { + restore(agent, get_link(node_id)); + } else { + state.key_buf().push_back((char)bases_[node_id]); + } + + if (node_id <= num_l1_nodes_) { + return; + } + node_id = louds_.select1(node_id) - node_id - 1; + } +} + +bool LoudsTrie::match_(Agent &agent, std::size_t node_id) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(node_id == 0, MARISA_RANGE_ERROR); + + State &state = agent.state(); + for ( ; ; ) { + const std::size_t cache_id = get_cache_id(node_id); + if (node_id == cache_[cache_id].child()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!match(agent, cache_[cache_id].link())) { + return false; + } + } else if (cache_[cache_id].label() == + agent.query()[state.query_pos()]) { + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + node_id = cache_[cache_id].parent(); + if (node_id == 0) { + return true; + } else if (state.query_pos() >= agent.query().length()) { + return false; + } + continue; + } + + if (link_flags_[node_id]) { + if (next_trie_.get() != NULL) { + if (!match(agent, get_link(node_id))) { + return false; + } + } else if (!tail_.match(agent, get_link(node_id))) { + return false; + } + } else if (bases_[node_id] == (UInt8)agent.query()[state.query_pos()]) { + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + if (node_id <= num_l1_nodes_) { + return true; + } else if (state.query_pos() >= agent.query().length()) { + return false; + } + node_id = louds_.select1(node_id) - node_id - 1; + } +} + +bool LoudsTrie::prefix_match_(Agent &agent, std::size_t node_id) const { + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(node_id == 0, MARISA_RANGE_ERROR); + + State &state = agent.state(); + for ( ; ; ) { + const std::size_t cache_id = get_cache_id(node_id); + if (node_id == cache_[cache_id].child()) { + if (cache_[cache_id].extra() != MARISA_INVALID_EXTRA) { + if (!prefix_match(agent, cache_[cache_id].link())) { + return false; + } + } else if (cache_[cache_id].label() == + agent.query()[state.query_pos()]) { + state.key_buf().push_back(cache_[cache_id].label()); + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + node_id = cache_[cache_id].parent(); + if (node_id == 0) { + return true; + } + } else { + if (link_flags_[node_id]) { + if (!prefix_match(agent, get_link(node_id))) { + return false; + } + } else if (bases_[node_id] == (UInt8)agent.query()[state.query_pos()]) { + state.key_buf().push_back((char)bases_[node_id]); + state.set_query_pos(state.query_pos() + 1); + } else { + return false; + } + + if (node_id <= num_l1_nodes_) { + return true; + } + node_id = louds_.select1(node_id) - node_id - 1; + } + + if (state.query_pos() >= agent.query().length()) { + restore_(agent, node_id); + return true; + } + } +} + +std::size_t LoudsTrie::get_cache_id(std::size_t node_id, char label) const { + return (node_id ^ (node_id << 5) ^ (UInt8)label) & cache_mask_; +} + +std::size_t LoudsTrie::get_cache_id(std::size_t node_id) const { + return node_id & cache_mask_; +} + +std::size_t LoudsTrie::get_link(std::size_t node_id) const { + return bases_[node_id] | (extras_[link_flags_.rank1(node_id)] * 256); +} + +std::size_t LoudsTrie::get_link(std::size_t node_id, + std::size_t link_id) const { + return bases_[node_id] | (extras_[link_id] * 256); +} + +std::size_t LoudsTrie::update_link_id(std::size_t link_id, + std::size_t node_id) const { + return (link_id == MARISA_INVALID_LINK_ID) ? + link_flags_.rank1(node_id) : (link_id + 1); +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/louds-trie.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/louds-trie.h new file mode 100644 index 00000000000..24ae013b25b --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/louds-trie.h @@ -0,0 +1,134 @@ +#ifndef MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_ +#define MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_ + +#include "marisa/keyset.h" +#include "marisa/agent.h" +#include "marisa/grimoire/vector.h" +#include "marisa/grimoire/trie/config.h" +#include "marisa/grimoire/trie/key.h" +#include "marisa/grimoire/trie/tail.h" +#include "marisa/grimoire/trie/cache.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class LoudsTrie { + public: + LoudsTrie(); + ~LoudsTrie(); + + void build(Keyset &keyset, int flags); + + void map(Mapper &mapper); + void read(Reader &reader); + void write(Writer &writer) const; + + bool lookup(Agent &agent) const; + void reverse_lookup(Agent &agent) const; + bool common_prefix_search(Agent &agent) const; + bool predictive_search(Agent &agent) const; + + std::size_t num_tries() const { + return config_.num_tries(); + } + std::size_t num_keys() const { + return size(); + } + std::size_t num_nodes() const { + return (louds_.size() / 2) - 1; + } + + CacheLevel cache_level() const { + return config_.cache_level(); + } + TailMode tail_mode() const { + return config_.tail_mode(); + } + NodeOrder node_order() const { + return config_.node_order(); + } + + bool empty() const { + return size() == 0; + } + std::size_t size() const { + return terminal_flags_.num_1s(); + } + std::size_t total_size() const; + std::size_t io_size() const; + + void clear(); + void swap(LoudsTrie &rhs); + + private: + BitVector louds_; + BitVector terminal_flags_; + BitVector link_flags_; + Vector<UInt8> bases_; + FlatVector extras_; + Tail tail_; + scoped_ptr<LoudsTrie> next_trie_; + Vector<Cache> cache_; + std::size_t cache_mask_; + std::size_t num_l1_nodes_; + Config config_; + Mapper mapper_; + + void build_(Keyset &keyset, const Config &config); + + template <typename T> + void build_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id); + template <typename T> + void build_current_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id); + template <typename T> + void build_next_trie(Vector<T> &keys, + Vector<UInt32> *terminals, const Config &config, std::size_t trie_id); + template <typename T> + void build_terminals(const Vector<T> &keys, + Vector<UInt32> *terminals) const; + + void reserve_cache(const Config &config, std::size_t trie_id, + std::size_t num_keys); + template <typename T> + void cache(std::size_t parent, std::size_t child, + float weight, char label); + void fill_cache(); + + void map_(Mapper &mapper); + void read_(Reader &reader); + void write_(Writer &writer) const; + + inline bool find_child(Agent &agent) const; + inline bool predictive_find_child(Agent &agent) const; + + inline void restore(Agent &agent, std::size_t node_id) const; + inline bool match(Agent &agent, std::size_t node_id) const; + inline bool prefix_match(Agent &agent, std::size_t node_id) const; + + void restore_(Agent &agent, std::size_t node_id) const; + bool match_(Agent &agent, std::size_t node_id) const; + bool prefix_match_(Agent &agent, std::size_t node_id) const; + + inline std::size_t get_cache_id(std::size_t node_id, char label) const; + inline std::size_t get_cache_id(std::size_t node_id) const; + + inline std::size_t get_link(std::size_t node_id) const; + inline std::size_t get_link(std::size_t node_id, + std::size_t link_id) const; + + inline std::size_t update_link_id(std::size_t link_id, + std::size_t node_id) const; + + // Disallows copy and assignment. + LoudsTrie(const LoudsTrie &); + LoudsTrie &operator=(const LoudsTrie &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/range.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/range.h new file mode 100644 index 00000000000..6c78ddbd11e --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/range.h @@ -0,0 +1,115 @@ +#ifndef MARISA_GRIMOIRE_TRIE_RANGE_H_ +#define MARISA_GRIMOIRE_TRIE_RANGE_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Range { + public: + Range() : begin_(0), end_(0), key_pos_(0) {} + + void set_begin(std::size_t begin) { + MARISA_DEBUG_IF(begin > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + begin_ = static_cast<UInt32>(begin); + } + void set_end(std::size_t end) { + MARISA_DEBUG_IF(end > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + end_ = static_cast<UInt32>(end); + } + void set_key_pos(std::size_t key_pos) { + MARISA_DEBUG_IF(key_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + key_pos_ = static_cast<UInt32>(key_pos); + } + + std::size_t begin() const { + return begin_; + } + std::size_t end() const { + return end_; + } + std::size_t key_pos() const { + return key_pos_; + } + + private: + UInt32 begin_; + UInt32 end_; + UInt32 key_pos_; +}; + +inline Range make_range(std::size_t begin, std::size_t end, + std::size_t key_pos) { + Range range; + range.set_begin(begin); + range.set_end(end); + range.set_key_pos(key_pos); + return range; +} + +class WeightedRange { + public: + WeightedRange() : range_(), weight_(0.0F) {} + + void set_range(const Range &range) { + range_ = range; + } + void set_begin(std::size_t begin) { + range_.set_begin(begin); + } + void set_end(std::size_t end) { + range_.set_end(end); + } + void set_key_pos(std::size_t key_pos) { + range_.set_key_pos(key_pos); + } + void set_weight(float weight) { + weight_ = weight; + } + + const Range &range() const { + return range_; + } + std::size_t begin() const { + return range_.begin(); + } + std::size_t end() const { + return range_.end(); + } + std::size_t key_pos() const { + return range_.key_pos(); + } + float weight() const { + return weight_; + } + + private: + Range range_; + float weight_; +}; + +inline bool operator<(const WeightedRange &lhs, const WeightedRange &rhs) { + return lhs.weight() < rhs.weight(); +} + +inline bool operator>(const WeightedRange &lhs, const WeightedRange &rhs) { + return lhs.weight() > rhs.weight(); +} + +inline WeightedRange make_weighted_range(std::size_t begin, std::size_t end, + std::size_t key_pos, float weight) { + WeightedRange range; + range.set_begin(begin); + range.set_end(end); + range.set_key_pos(key_pos); + range.set_weight(weight); + return range; +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_RANGE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/state.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/state.h new file mode 100644 index 00000000000..df605a6a3e5 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/state.h @@ -0,0 +1,117 @@ +#ifndef MARISA_GRIMOIRE_TRIE_STATE_H_ +#define MARISA_GRIMOIRE_TRIE_STATE_H_ + +#include "marisa/grimoire/vector.h" +#include "marisa/grimoire/trie/history.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +// A search agent has its internal state and the status codes are defined +// below. +typedef enum StatusCode { + MARISA_READY_TO_ALL, + MARISA_READY_TO_COMMON_PREFIX_SEARCH, + MARISA_READY_TO_PREDICTIVE_SEARCH, + MARISA_END_OF_COMMON_PREFIX_SEARCH, + MARISA_END_OF_PREDICTIVE_SEARCH, +} StatusCode; + +class State { + public: + State() + : key_buf_(), history_(), node_id_(0), query_pos_(0), + history_pos_(0), status_code_(MARISA_READY_TO_ALL) {} + + void set_node_id(std::size_t node_id) { + MARISA_DEBUG_IF(node_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + node_id_ = (UInt32)node_id; + } + void set_query_pos(std::size_t query_pos) { + MARISA_DEBUG_IF(query_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + query_pos_ = (UInt32)query_pos; + } + void set_history_pos(std::size_t history_pos) { + MARISA_DEBUG_IF(history_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + history_pos_ = (UInt32)history_pos; + } + void set_status_code(StatusCode status_code) { + status_code_ = status_code; + } + + std::size_t node_id() const { + return node_id_; + } + std::size_t query_pos() const { + return query_pos_; + } + std::size_t history_pos() const { + return history_pos_; + } + StatusCode status_code() const { + return status_code_; + } + + const Vector<char> &key_buf() const { + return key_buf_; + } + const Vector<History> &history() const { + return history_; + } + + Vector<char> &key_buf() { + return key_buf_; + } + Vector<History> &history() { + return history_; + } + + void reset() { + status_code_ = MARISA_READY_TO_ALL; + } + + void lookup_init() { + node_id_ = 0; + query_pos_ = 0; + status_code_ = MARISA_READY_TO_ALL; + } + void reverse_lookup_init() { + key_buf_.resize(0); + key_buf_.reserve(32); + status_code_ = MARISA_READY_TO_ALL; + } + void common_prefix_search_init() { + node_id_ = 0; + query_pos_ = 0; + status_code_ = MARISA_READY_TO_COMMON_PREFIX_SEARCH; + } + void predictive_search_init() { + key_buf_.resize(0); + key_buf_.reserve(64); + history_.resize(0); + history_.reserve(4); + node_id_ = 0; + query_pos_ = 0; + history_pos_ = 0; + status_code_ = MARISA_READY_TO_PREDICTIVE_SEARCH; + } + + private: + Vector<char> key_buf_; + Vector<History> history_; + UInt32 node_id_; + UInt32 query_pos_; + UInt32 history_pos_; + StatusCode status_code_; + + // Disallows copy and assignment. + State(const State &); + State &operator=(const State &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_STATE_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/tail.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/tail.cc new file mode 100644 index 00000000000..bd9bd01d56e --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/tail.cc @@ -0,0 +1,218 @@ +#include "marisa/grimoire/algorithm.h" +#include "marisa/grimoire/trie/state.h" +#include "marisa/grimoire/trie/tail.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +Tail::Tail() : buf_(), end_flags_() {} + +void Tail::build(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode) { + MARISA_THROW_IF(offsets == NULL, MARISA_NULL_ERROR); + + switch (mode) { + case MARISA_TEXT_TAIL: { + for (std::size_t i = 0; i < entries.size(); ++i) { + const char * const ptr = entries[i].ptr(); + const std::size_t length = entries[i].length(); + for (std::size_t j = 0; j < length; ++j) { + if (ptr[j] == '\0') { + mode = MARISA_BINARY_TAIL; + break; + } + } + if (mode == MARISA_BINARY_TAIL) { + break; + } + } + break; + } + case MARISA_BINARY_TAIL: { + break; + } + default: { + MARISA_THROW(MARISA_CODE_ERROR, "undefined tail mode"); + } + } + + Tail temp; + temp.build_(entries, offsets, mode); + swap(temp); +} + +void Tail::map(Mapper &mapper) { + Tail temp; + temp.map_(mapper); + swap(temp); +} + +void Tail::read(Reader &reader) { + Tail temp; + temp.read_(reader); + swap(temp); +} + +void Tail::write(Writer &writer) const { + write_(writer); +} + +void Tail::restore(Agent &agent, std::size_t offset) const { + MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (end_flags_.empty()) { + for (const char *ptr = &buf_[offset]; *ptr != '\0'; ++ptr) { + state.key_buf().push_back(*ptr); + } + } else { + do { + state.key_buf().push_back(buf_[offset]); + } while (!end_flags_[offset++]); + } +} + +bool Tail::match(Agent &agent, std::size_t offset) const { + MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(), + MARISA_BOUND_ERROR); + + State &state = agent.state(); + if (end_flags_.empty()) { + const char * const ptr = &buf_[offset] - state.query_pos(); + do { + if (ptr[state.query_pos()] != agent.query()[state.query_pos()]) { + return false; + } + state.set_query_pos(state.query_pos() + 1); + if (ptr[state.query_pos()] == '\0') { + return true; + } + } while (state.query_pos() < agent.query().length()); + return false; + } else { + do { + if (buf_[offset] != agent.query()[state.query_pos()]) { + return false; + } + state.set_query_pos(state.query_pos() + 1); + if (end_flags_[offset++]) { + return true; + } + } while (state.query_pos() < agent.query().length()); + return false; + } +} + +bool Tail::prefix_match(Agent &agent, std::size_t offset) const { + MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR); + + State &state = agent.state(); + if (end_flags_.empty()) { + const char *ptr = &buf_[offset] - state.query_pos(); + do { + if (ptr[state.query_pos()] != agent.query()[state.query_pos()]) { + return false; + } + state.key_buf().push_back(ptr[state.query_pos()]); + state.set_query_pos(state.query_pos() + 1); + if (ptr[state.query_pos()] == '\0') { + return true; + } + } while (state.query_pos() < agent.query().length()); + ptr += state.query_pos(); + do { + state.key_buf().push_back(*ptr); + } while (*++ptr != '\0'); + return true; + } else { + do { + if (buf_[offset] != agent.query()[state.query_pos()]) { + return false; + } + state.key_buf().push_back(buf_[offset]); + state.set_query_pos(state.query_pos() + 1); + if (end_flags_[offset++]) { + return true; + } + } while (state.query_pos() < agent.query().length()); + do { + state.key_buf().push_back(buf_[offset]); + } while (!end_flags_[offset++]); + return true; + } +} + +void Tail::clear() { + Tail().swap(*this); +} + +void Tail::swap(Tail &rhs) { + buf_.swap(rhs.buf_); + end_flags_.swap(rhs.end_flags_); +} + +void Tail::build_(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode) { + for (std::size_t i = 0; i < entries.size(); ++i) { + entries[i].set_id(i); + } + Algorithm().sort(entries.begin(), entries.end()); + + Vector<UInt32> temp_offsets; + temp_offsets.resize(entries.size(), 0); + + const Entry dummy; + const Entry *last = &dummy; + for (std::size_t i = entries.size(); i > 0; --i) { + const Entry ¤t = entries[i - 1]; + MARISA_THROW_IF(current.length() == 0, MARISA_RANGE_ERROR); + std::size_t match = 0; + while ((match < current.length()) && (match < last->length()) && + ((*last)[match] == current[match])) { + ++match; + } + if ((match == current.length()) && (last->length() != 0)) { + temp_offsets[current.id()] = (UInt32)( + temp_offsets[last->id()] + (last->length() - match)); + } else { + temp_offsets[current.id()] = (UInt32)buf_.size(); + for (std::size_t j = 1; j <= current.length(); ++j) { + buf_.push_back(current[current.length() - j]); + } + if (mode == MARISA_TEXT_TAIL) { + buf_.push_back('\0'); + } else { + for (std::size_t j = 1; j < current.length(); ++j) { + end_flags_.push_back(false); + } + end_flags_.push_back(true); + } + MARISA_THROW_IF(buf_.size() > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + } + last = ¤t; + } + buf_.shrink(); + + offsets->swap(temp_offsets); +} + +void Tail::map_(Mapper &mapper) { + buf_.map(mapper); + end_flags_.map(mapper); +} + +void Tail::read_(Reader &reader) { + buf_.read(reader); + end_flags_.read(reader); +} + +void Tail::write_(Writer &writer) const { + buf_.write(writer); + end_flags_.write(writer); +} + +} // namespace trie +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/tail.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/tail.h new file mode 100644 index 00000000000..dd24f3ef81f --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/trie/tail.h @@ -0,0 +1,72 @@ +#ifndef MARISA_GRIMOIRE_TRIE_TAIL_H_ +#define MARISA_GRIMOIRE_TRIE_TAIL_H_ + +#include "marisa/agent.h" +#include "marisa/grimoire/vector.h" +#include "marisa/grimoire/trie/entry.h" + +namespace marisa { +namespace grimoire { +namespace trie { + +class Tail { + public: + Tail(); + + void build(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode); + + void map(Mapper &mapper); + void read(Reader &reader); + void write(Writer &writer) const; + + void restore(Agent &agent, std::size_t offset) const; + bool match(Agent &agent, std::size_t offset) const; + bool prefix_match(Agent &agent, std::size_t offset) const; + + const char &operator[](std::size_t offset) const { + MARISA_DEBUG_IF(offset >= buf_.size(), MARISA_BOUND_ERROR); + return buf_[offset]; + } + + TailMode mode() const { + return end_flags_.empty() ? MARISA_TEXT_TAIL : MARISA_BINARY_TAIL; + } + + bool empty() const { + return buf_.empty(); + } + std::size_t size() const { + return buf_.size(); + } + std::size_t total_size() const { + return buf_.total_size() + end_flags_.total_size(); + } + std::size_t io_size() const { + return buf_.io_size() + end_flags_.io_size(); + } + + void clear(); + void swap(Tail &rhs); + + private: + Vector<char> buf_; + BitVector end_flags_; + + void build_(Vector<Entry> &entries, Vector<UInt32> *offsets, + TailMode mode); + + void map_(Mapper &mapper); + void read_(Reader &reader); + void write_(Writer &writer) const; + + // Disallows copy and assignment. + Tail(const Tail &); + Tail &operator=(const Tail &); +}; + +} // namespace trie +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_TRIE_TAIL_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector.h new file mode 100644 index 00000000000..582160baf99 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector.h @@ -0,0 +1,18 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_H_ + +#include "marisa/grimoire/vector/vector.h" +#include "marisa/grimoire/vector/flat-vector.h" +#include "marisa/grimoire/vector/bit-vector.h" + +namespace marisa { +namespace grimoire { + +using vector::Vector; +typedef vector::FlatVector FlatVector; +typedef vector::BitVector BitVector; + +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc new file mode 100644 index 00000000000..60ded67fe63 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc @@ -0,0 +1,844 @@ +#include "marisa/grimoire/vector/pop-count.h" +#include "marisa/grimoire/vector/bit-vector.h" + +namespace marisa { +namespace grimoire { +namespace vector { +namespace { + +#ifdef MARISA_USE_BMI2 +std::size_t select_bit(std::size_t i, std::size_t bit_id, UInt64 unit) { + #ifdef _MSC_VER + unsigned long pos; + ::_BitScanForward64(&pos, _pdep_u64(1ULL << i, unit)); + return bit_id + pos; + #else // _MSC_VER + return bit_id + ::__builtin_ctzll(_pdep_u64(1ULL << i, unit)); + #endif // _MSC_VER +} +#else // MARISA_USE_BMI2 +const UInt8 SELECT_TABLE[8][256] = { + { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 + }, + { + 7, 7, 7, 1, 7, 2, 2, 1, 7, 3, 3, 1, 3, 2, 2, 1, + 7, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 6, 6, 1, 6, 2, 2, 1, 6, 3, 3, 1, 3, 2, 2, 1, + 6, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 6, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 7, 7, 1, 7, 2, 2, 1, 7, 3, 3, 1, 3, 2, 2, 1, + 7, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 7, 6, 6, 1, 6, 2, 2, 1, 6, 3, 3, 1, 3, 2, 2, 1, + 6, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1, + 6, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1, + 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1 + }, + { + 7, 7, 7, 7, 7, 7, 7, 2, 7, 7, 7, 3, 7, 3, 3, 2, + 7, 7, 7, 4, 7, 4, 4, 2, 7, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 5, 7, 5, 5, 2, 7, 5, 5, 3, 5, 3, 3, 2, + 7, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 6, 7, 6, 6, 2, 7, 6, 6, 3, 6, 3, 3, 2, + 7, 6, 6, 4, 6, 4, 4, 2, 6, 4, 4, 3, 4, 3, 3, 2, + 7, 6, 6, 5, 6, 5, 5, 2, 6, 5, 5, 3, 5, 3, 3, 2, + 6, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 7, 7, 7, 7, 2, 7, 7, 7, 3, 7, 3, 3, 2, + 7, 7, 7, 4, 7, 4, 4, 2, 7, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 5, 7, 5, 5, 2, 7, 5, 5, 3, 5, 3, 3, 2, + 7, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2, + 7, 7, 7, 6, 7, 6, 6, 2, 7, 6, 6, 3, 6, 3, 3, 2, + 7, 6, 6, 4, 6, 4, 4, 2, 6, 4, 4, 3, 4, 3, 3, 2, + 7, 6, 6, 5, 6, 5, 5, 2, 6, 5, 5, 3, 5, 3, 3, 2, + 6, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 3, + 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 4, 7, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 3, + 7, 7, 7, 5, 7, 5, 5, 4, 7, 5, 5, 4, 5, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 3, + 7, 7, 7, 6, 7, 6, 6, 4, 7, 6, 6, 4, 6, 4, 4, 3, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 3, + 7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 3, + 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 4, 7, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 3, + 7, 7, 7, 5, 7, 5, 5, 4, 7, 5, 5, 4, 5, 4, 4, 3, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 3, + 7, 7, 7, 6, 7, 6, 6, 4, 7, 6, 6, 4, 6, 4, 4, 3, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 3, + 7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 4, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 5, 7, 7, 7, 5, 7, 5, 5, 4, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 4, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5, + 7, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 4 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 6, 7, 7, 7, 6, 7, 6, 6, 5 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6 + }, + { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 + } +}; + + #if MARISA_WORD_SIZE == 64 +const UInt64 MASK_01 = 0x0101010101010101ULL; + #if !defined(MARISA_X64) || !defined(MARISA_USE_SSSE3) +const UInt64 MASK_0F = 0x0F0F0F0F0F0F0F0FULL; +const UInt64 MASK_33 = 0x3333333333333333ULL; +const UInt64 MASK_55 = 0x5555555555555555ULL; + #endif // !defined(MARISA_X64) || !defined(MARISA_USE_SSSE3) + #if !defined(MARISA_X64) || !defined(MARISA_USE_POPCNT) +const UInt64 MASK_80 = 0x8080808080808080ULL; + #endif // !defined(MARISA_X64) || !defined(MARISA_USE_POPCNT) + +std::size_t select_bit(std::size_t i, std::size_t bit_id, UInt64 unit) { + UInt64 counts; + { + #if defined(MARISA_X64) && defined(MARISA_USE_SSSE3) + __m128i lower_nibbles = _mm_cvtsi64_si128( + static_cast<long long>(unit & 0x0F0F0F0F0F0F0F0FULL)); + __m128i upper_nibbles = _mm_cvtsi64_si128( + static_cast<long long>(unit & 0xF0F0F0F0F0F0F0F0ULL)); + upper_nibbles = _mm_srli_epi32(upper_nibbles, 4); + + __m128i lower_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + lower_counts = _mm_shuffle_epi8(lower_counts, lower_nibbles); + __m128i upper_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + upper_counts = _mm_shuffle_epi8(upper_counts, upper_nibbles); + + counts = static_cast<UInt64>(_mm_cvtsi128_si64( + _mm_add_epi8(lower_counts, upper_counts))); + #else // defined(MARISA_X64) && defined(MARISA_USE_SSSE3) + counts = unit - ((unit >> 1) & MASK_55); + counts = (counts & MASK_33) + ((counts >> 2) & MASK_33); + counts = (counts + (counts >> 4)) & MASK_0F; + #endif // defined(MARISA_X64) && defined(MARISA_USE_SSSE3) + counts *= MASK_01; + } + + #if defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + UInt8 skip; + { + __m128i x = _mm_cvtsi64_si128(static_cast<long long>((i + 1) * MASK_01)); + __m128i y = _mm_cvtsi64_si128(static_cast<long long>(counts)); + x = _mm_cmpgt_epi8(x, y); + skip = (UInt8)PopCount::count(static_cast<UInt64>(_mm_cvtsi128_si64(x))); + } + #else // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + const UInt64 x = (counts | MASK_80) - ((i + 1) * MASK_01); + #ifdef _MSC_VER + unsigned long skip; + ::_BitScanForward64(&skip, (x & MASK_80) >> 7); + #else // _MSC_VER + const int skip = ::__builtin_ctzll((x & MASK_80) >> 7); + #endif // _MSC_VER + #endif // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + + bit_id += static_cast<std::size_t>(skip); + unit >>= skip; + i -= ((counts << 8) >> skip) & 0xFF; + + return bit_id + SELECT_TABLE[i][unit & 0xFF]; +} + #else // MARISA_WORD_SIZE == 64 + #ifdef MARISA_USE_SSE2 +const UInt8 POPCNT_TABLE[256] = { + 0, 8, 8, 16, 8, 16, 16, 24, 8, 16, 16, 24, 16, 24, 24, 32, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 8, 16, 16, 24, 16, 24, 24, 32, 16, 24, 24, 32, 24, 32, 32, 40, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 16, 24, 24, 32, 24, 32, 32, 40, 24, 32, 32, 40, 32, 40, 40, 48, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 24, 32, 32, 40, 32, 40, 40, 48, 32, 40, 40, 48, 40, 48, 48, 56, + 32, 40, 40, 48, 40, 48, 48, 56, 40, 48, 48, 56, 48, 56, 56, 64 +}; + +std::size_t select_bit(std::size_t i, std::size_t bit_id, + UInt32 unit_lo, UInt32 unit_hi) { + __m128i unit; + { + __m128i lower_dword = _mm_cvtsi32_si128(unit_lo); + __m128i upper_dword = _mm_cvtsi32_si128(unit_hi); + upper_dword = _mm_slli_si128(upper_dword, 4); + unit = _mm_or_si128(lower_dword, upper_dword); + } + + __m128i counts; + { + #ifdef MARISA_USE_SSSE3 + __m128i lower_nibbles = _mm_set1_epi8(0x0F); + lower_nibbles = _mm_and_si128(lower_nibbles, unit); + __m128i upper_nibbles = _mm_set1_epi8((UInt8)0xF0); + upper_nibbles = _mm_and_si128(upper_nibbles, unit); + upper_nibbles = _mm_srli_epi32(upper_nibbles, 4); + + __m128i lower_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + lower_counts = _mm_shuffle_epi8(lower_counts, lower_nibbles); + __m128i upper_counts = + _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + upper_counts = _mm_shuffle_epi8(upper_counts, upper_nibbles); + + counts = _mm_add_epi8(lower_counts, upper_counts); + #else // MARISA_USE_SSSE3 + __m128i x = _mm_srli_epi32(unit, 1); + x = _mm_and_si128(x, _mm_set1_epi8(0x55)); + x = _mm_sub_epi8(unit, x); + + __m128i y = _mm_srli_epi32(x, 2); + y = _mm_and_si128(y, _mm_set1_epi8(0x33)); + x = _mm_and_si128(x, _mm_set1_epi8(0x33)); + x = _mm_add_epi8(x, y); + + y = _mm_srli_epi32(x, 4); + x = _mm_add_epi8(x, y); + counts = _mm_and_si128(x, _mm_set1_epi8(0x0F)); + #endif // MARISA_USE_SSSE3 + } + + __m128i accumulated_counts; + { + __m128i x = counts; + x = _mm_slli_si128(x, 1); + __m128i y = counts; + y = _mm_add_epi32(y, x); + + x = y; + y = _mm_slli_si128(y, 2); + x = _mm_add_epi32(x, y); + + y = x; + x = _mm_slli_si128(x, 4); + y = _mm_add_epi32(y, x); + + accumulated_counts = _mm_set_epi32(0x7F7F7F7FU, 0x7F7F7F7FU, 0, 0); + accumulated_counts = _mm_or_si128(accumulated_counts, y); + } + + UInt8 skip; + { + __m128i x = _mm_set1_epi8((UInt8)(i + 1)); + x = _mm_cmpgt_epi8(x, accumulated_counts); + skip = POPCNT_TABLE[_mm_movemask_epi8(x)]; + } + + UInt8 byte; + { + #ifdef _MSC_VER + __declspec(align(16)) UInt8 unit_bytes[16]; + __declspec(align(16)) UInt8 accumulated_counts_bytes[16]; + #else // _MSC_VER + UInt8 unit_bytes[16] __attribute__ ((aligned (16))); + UInt8 accumulated_counts_bytes[16] __attribute__ ((aligned (16))); + #endif // _MSC_VER + accumulated_counts = _mm_slli_si128(accumulated_counts, 1); + _mm_store_si128(reinterpret_cast<__m128i *>(unit_bytes), unit); + _mm_store_si128(reinterpret_cast<__m128i *>(accumulated_counts_bytes), + accumulated_counts); + + bit_id += skip; + byte = unit_bytes[skip / 8]; + i -= accumulated_counts_bytes[skip / 8]; + } + + return bit_id + SELECT_TABLE[i][byte]; +} + #endif // MARISA_USE_SSE2 + #endif // MARISA_WORD_SIZE == 64 +#endif // MARISA_USE_BMI2 + +} // namespace + +#if MARISA_WORD_SIZE == 64 + +std::size_t BitVector::rank1(std::size_t i) const { + MARISA_DEBUG_IF(ranks_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i > size_, MARISA_BOUND_ERROR); + + const RankIndex &rank = ranks_[i / 512]; + std::size_t offset = rank.abs(); + switch ((i / 64) % 8) { + case 1: { + offset += rank.rel1(); + break; + } + case 2: { + offset += rank.rel2(); + break; + } + case 3: { + offset += rank.rel3(); + break; + } + case 4: { + offset += rank.rel4(); + break; + } + case 5: { + offset += rank.rel5(); + break; + } + case 6: { + offset += rank.rel6(); + break; + } + case 7: { + offset += rank.rel7(); + break; + } + } + offset += PopCount::count(units_[i / 64] & ((1ULL << (i % 64)) - 1)); + return offset; +} + +std::size_t BitVector::select0(std::size_t i) const { + MARISA_DEBUG_IF(select0s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_0s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select0s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select0s_[select_id]; + } + std::size_t begin = select0s_[select_id] / 512; + std::size_t end = (select0s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ((begin + 1) * 512) - ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < (middle * 512) - ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= (rank_id * 512) - ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 8; + if (i < (256U - rank.rel4())) { + if (i < (128U - rank.rel2())) { + if (i >= (64U - rank.rel1())) { + unit_id += 1; + i -= 64 - rank.rel1(); + } + } else if (i < (192U - rank.rel3())) { + unit_id += 2; + i -= 128 - rank.rel2(); + } else { + unit_id += 3; + i -= 192 - rank.rel3(); + } + } else if (i < (384U - rank.rel6())) { + if (i < (320U - rank.rel5())) { + unit_id += 4; + i -= 256 - rank.rel4(); + } else { + unit_id += 5; + i -= 320 - rank.rel5(); + } + } else if (i < (448U - rank.rel7())) { + unit_id += 6; + i -= 384 - rank.rel6(); + } else { + unit_id += 7; + i -= 448 - rank.rel7(); + } + + return select_bit(i, unit_id * 64, ~units_[unit_id]); +} + +std::size_t BitVector::select1(std::size_t i) const { + MARISA_DEBUG_IF(select1s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_1s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select1s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select1s_[select_id]; + } + std::size_t begin = select1s_[select_id] / 512; + std::size_t end = (select1s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 8; + if (i < rank.rel4()) { + if (i < rank.rel2()) { + if (i >= rank.rel1()) { + unit_id += 1; + i -= rank.rel1(); + } + } else if (i < rank.rel3()) { + unit_id += 2; + i -= rank.rel2(); + } else { + unit_id += 3; + i -= rank.rel3(); + } + } else if (i < rank.rel6()) { + if (i < rank.rel5()) { + unit_id += 4; + i -= rank.rel4(); + } else { + unit_id += 5; + i -= rank.rel5(); + } + } else if (i < rank.rel7()) { + unit_id += 6; + i -= rank.rel6(); + } else { + unit_id += 7; + i -= rank.rel7(); + } + + return select_bit(i, unit_id * 64, units_[unit_id]); +} + +#else // MARISA_WORD_SIZE == 64 + +std::size_t BitVector::rank1(std::size_t i) const { + MARISA_DEBUG_IF(ranks_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i > size_, MARISA_BOUND_ERROR); + + const RankIndex &rank = ranks_[i / 512]; + std::size_t offset = rank.abs(); + switch ((i / 64) % 8) { + case 1: { + offset += rank.rel1(); + break; + } + case 2: { + offset += rank.rel2(); + break; + } + case 3: { + offset += rank.rel3(); + break; + } + case 4: { + offset += rank.rel4(); + break; + } + case 5: { + offset += rank.rel5(); + break; + } + case 6: { + offset += rank.rel6(); + break; + } + case 7: { + offset += rank.rel7(); + break; + } + } + if (((i / 32) & 1) == 1) { + offset += PopCount::count(units_[(i / 32) - 1]); + } + offset += PopCount::count(units_[i / 32] & ((1U << (i % 32)) - 1)); + return offset; +} + +std::size_t BitVector::select0(std::size_t i) const { + MARISA_DEBUG_IF(select0s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_0s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select0s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select0s_[select_id]; + } + std::size_t begin = select0s_[select_id] / 512; + std::size_t end = (select0s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ((begin + 1) * 512) - ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < (middle * 512) - ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= (rank_id * 512) - ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 16; + if (i < (256U - rank.rel4())) { + if (i < (128U - rank.rel2())) { + if (i >= (64U - rank.rel1())) { + unit_id += 2; + i -= 64 - rank.rel1(); + } + } else if (i < (192U - rank.rel3())) { + unit_id += 4; + i -= 128 - rank.rel2(); + } else { + unit_id += 6; + i -= 192 - rank.rel3(); + } + } else if (i < (384U - rank.rel6())) { + if (i < (320U - rank.rel5())) { + unit_id += 8; + i -= 256 - rank.rel4(); + } else { + unit_id += 10; + i -= 320 - rank.rel5(); + } + } else if (i < (448U - rank.rel7())) { + unit_id += 12; + i -= 384 - rank.rel6(); + } else { + unit_id += 14; + i -= 448 - rank.rel7(); + } + +#ifdef MARISA_USE_SSE2 + return select_bit(i, unit_id * 32, ~units_[unit_id], ~units_[unit_id + 1]); +#else // MARISA_USE_SSE2 + UInt32 unit = ~units_[unit_id]; + PopCount count(unit); + if (i >= count.lo32()) { + ++unit_id; + i -= count.lo32(); + unit = ~units_[unit_id]; + count = PopCount(unit); + } + + std::size_t bit_id = unit_id * 32; + if (i < count.lo16()) { + if (i >= count.lo8()) { + bit_id += 8; + unit >>= 8; + i -= count.lo8(); + } + } else if (i < count.lo24()) { + bit_id += 16; + unit >>= 16; + i -= count.lo16(); + } else { + bit_id += 24; + unit >>= 24; + i -= count.lo24(); + } + return bit_id + SELECT_TABLE[i][unit & 0xFF]; +#endif // MARISA_USE_SSE2 +} + +std::size_t BitVector::select1(std::size_t i) const { + MARISA_DEBUG_IF(select1s_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= num_1s(), MARISA_BOUND_ERROR); + + const std::size_t select_id = i / 512; + MARISA_DEBUG_IF((select_id + 1) >= select1s_.size(), MARISA_BOUND_ERROR); + if ((i % 512) == 0) { + return select1s_[select_id]; + } + std::size_t begin = select1s_[select_id] / 512; + std::size_t end = (select1s_[select_id + 1] + 511) / 512; + if (begin + 10 >= end) { + while (i >= ranks_[begin + 1].abs()) { + ++begin; + } + } else { + while (begin + 1 < end) { + const std::size_t middle = (begin + end) / 2; + if (i < ranks_[middle].abs()) { + end = middle; + } else { + begin = middle; + } + } + } + const std::size_t rank_id = begin; + i -= ranks_[rank_id].abs(); + + const RankIndex &rank = ranks_[rank_id]; + std::size_t unit_id = rank_id * 16; + if (i < rank.rel4()) { + if (i < rank.rel2()) { + if (i >= rank.rel1()) { + unit_id += 2; + i -= rank.rel1(); + } + } else if (i < rank.rel3()) { + unit_id += 4; + i -= rank.rel2(); + } else { + unit_id += 6; + i -= rank.rel3(); + } + } else if (i < rank.rel6()) { + if (i < rank.rel5()) { + unit_id += 8; + i -= rank.rel4(); + } else { + unit_id += 10; + i -= rank.rel5(); + } + } else if (i < rank.rel7()) { + unit_id += 12; + i -= rank.rel6(); + } else { + unit_id += 14; + i -= rank.rel7(); + } + +#ifdef MARISA_USE_SSE2 + return select_bit(i, unit_id * 32, units_[unit_id], units_[unit_id + 1]); +#else // MARISA_USE_SSE2 + UInt32 unit = units_[unit_id]; + PopCount count(unit); + if (i >= count.lo32()) { + ++unit_id; + i -= count.lo32(); + unit = units_[unit_id]; + count = PopCount(unit); + } + + std::size_t bit_id = unit_id * 32; + if (i < count.lo16()) { + if (i >= count.lo8()) { + bit_id += 8; + unit >>= 8; + i -= count.lo8(); + } + } else if (i < count.lo24()) { + bit_id += 16; + unit >>= 16; + i -= count.lo16(); + } else { + bit_id += 24; + unit >>= 24; + i -= count.lo24(); + } + return bit_id + SELECT_TABLE[i][unit & 0xFF]; +#endif // MARISA_USE_SSE2 +} + +#endif // MARISA_WORD_SIZE == 64 + +void BitVector::build_index(const BitVector &bv, + bool enables_select0, bool enables_select1) { + ranks_.resize((bv.size() / 512) + (((bv.size() % 512) != 0) ? 1 : 0) + 1); + + std::size_t num_0s = 0; + std::size_t num_1s = 0; + + for (std::size_t i = 0; i < bv.size(); ++i) { + if ((i % 64) == 0) { + const std::size_t rank_id = i / 512; + switch ((i / 64) % 8) { + case 0: { + ranks_[rank_id].set_abs(num_1s); + break; + } + case 1: { + ranks_[rank_id].set_rel1(num_1s - ranks_[rank_id].abs()); + break; + } + case 2: { + ranks_[rank_id].set_rel2(num_1s - ranks_[rank_id].abs()); + break; + } + case 3: { + ranks_[rank_id].set_rel3(num_1s - ranks_[rank_id].abs()); + break; + } + case 4: { + ranks_[rank_id].set_rel4(num_1s - ranks_[rank_id].abs()); + break; + } + case 5: { + ranks_[rank_id].set_rel5(num_1s - ranks_[rank_id].abs()); + break; + } + case 6: { + ranks_[rank_id].set_rel6(num_1s - ranks_[rank_id].abs()); + break; + } + case 7: { + ranks_[rank_id].set_rel7(num_1s - ranks_[rank_id].abs()); + break; + } + } + } + + if (bv[i]) { + if (enables_select1 && ((num_1s % 512) == 0)) { + select1s_.push_back(static_cast<UInt32>(i)); + } + ++num_1s; + } else { + if (enables_select0 && ((num_0s % 512) == 0)) { + select0s_.push_back(static_cast<UInt32>(i)); + } + ++num_0s; + } + } + + if ((bv.size() % 512) != 0) { + const std::size_t rank_id = (bv.size() - 1) / 512; + switch (((bv.size() - 1) / 64) % 8) { + case 0: { + ranks_[rank_id].set_rel1(num_1s - ranks_[rank_id].abs()); + } // fall through + case 1: { + ranks_[rank_id].set_rel2(num_1s - ranks_[rank_id].abs()); + } // fall through + case 2: { + ranks_[rank_id].set_rel3(num_1s - ranks_[rank_id].abs()); + } // fall through + case 3: { + ranks_[rank_id].set_rel4(num_1s - ranks_[rank_id].abs()); + } // fall through + case 4: { + ranks_[rank_id].set_rel5(num_1s - ranks_[rank_id].abs()); + } // fall through + case 5: { + ranks_[rank_id].set_rel6(num_1s - ranks_[rank_id].abs()); + } // fall through + case 6: { + ranks_[rank_id].set_rel7(num_1s - ranks_[rank_id].abs()); + break; + } + } + } + + size_ = bv.size(); + num_1s_ = bv.num_1s(); + + ranks_.back().set_abs(num_1s); + if (enables_select0) { + select0s_.push_back(static_cast<UInt32>(bv.size())); + select0s_.shrink(); + } + if (enables_select1) { + select1s_.push_back(static_cast<UInt32>(bv.size())); + select1s_.shrink(); + } +} + +} // namespace vector +} // namespace grimoire +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/bit-vector.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/bit-vector.h new file mode 100644 index 00000000000..ea698f13311 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/bit-vector.h @@ -0,0 +1,179 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_BIT_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_BIT_VECTOR_H_ + +#include "marisa/grimoire/vector/rank-index.h" +#include "marisa/grimoire/vector/vector.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +class BitVector { + public: +#if MARISA_WORD_SIZE == 64 + typedef UInt64 Unit; +#else // MARISA_WORD_SIZE == 64 + typedef UInt32 Unit; +#endif // MARISA_WORD_SIZE == 64 + + BitVector() + : units_(), size_(0), num_1s_(0), ranks_(), select0s_(), select1s_() {} + + void build(bool enables_select0, bool enables_select1) { + BitVector temp; + temp.build_index(*this, enables_select0, enables_select1); + units_.shrink(); + temp.units_.swap(units_); + swap(temp); + } + + void map(Mapper &mapper) { + BitVector temp; + temp.map_(mapper); + swap(temp); + } + void read(Reader &reader) { + BitVector temp; + temp.read_(reader); + swap(temp); + } + void write(Writer &writer) const { + write_(writer); + } + + void disable_select0() { + select0s_.clear(); + } + void disable_select1() { + select1s_.clear(); + } + + void push_back(bool bit) { + MARISA_THROW_IF(size_ == MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + if (size_ == (MARISA_WORD_SIZE * units_.size())) { + units_.resize(units_.size() + (64 / MARISA_WORD_SIZE), 0); + } + if (bit) { + units_[size_ / MARISA_WORD_SIZE] |= + (Unit)1 << (size_ % MARISA_WORD_SIZE); + ++num_1s_; + } + ++size_; + } + + bool operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return (units_[i / MARISA_WORD_SIZE] + & ((Unit)1 << (i % MARISA_WORD_SIZE))) != 0; + } + + std::size_t rank0(std::size_t i) const { + MARISA_DEBUG_IF(ranks_.empty(), MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i > size_, MARISA_BOUND_ERROR); + return i - rank1(i); + } + std::size_t rank1(std::size_t i) const; + + std::size_t select0(std::size_t i) const; + std::size_t select1(std::size_t i) const; + + std::size_t num_0s() const { + return size_ - num_1s_; + } + std::size_t num_1s() const { + return num_1s_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + std::size_t total_size() const { + return units_.total_size() + ranks_.total_size() + + select0s_.total_size() + select1s_.total_size(); + } + std::size_t io_size() const { + return units_.io_size() + (sizeof(UInt32) * 2) + ranks_.io_size() + + select0s_.io_size() + select1s_.io_size(); + } + + void clear() { + BitVector().swap(*this); + } + void swap(BitVector &rhs) { + units_.swap(rhs.units_); + marisa::swap(size_, rhs.size_); + marisa::swap(num_1s_, rhs.num_1s_); + ranks_.swap(rhs.ranks_); + select0s_.swap(rhs.select0s_); + select1s_.swap(rhs.select1s_); + } + + private: + Vector<Unit> units_; + std::size_t size_; + std::size_t num_1s_; + Vector<RankIndex> ranks_; + Vector<UInt32> select0s_; + Vector<UInt32> select1s_; + + void build_index(const BitVector &bv, + bool enables_select0, bool enables_select1); + + void map_(Mapper &mapper) { + units_.map(mapper); + { + UInt32 temp_size; + mapper.map(&temp_size); + size_ = temp_size; + } + { + UInt32 temp_num_1s; + mapper.map(&temp_num_1s); + MARISA_THROW_IF(temp_num_1s > size_, MARISA_FORMAT_ERROR); + num_1s_ = temp_num_1s; + } + ranks_.map(mapper); + select0s_.map(mapper); + select1s_.map(mapper); + } + + void read_(Reader &reader) { + units_.read(reader); + { + UInt32 temp_size; + reader.read(&temp_size); + size_ = temp_size; + } + { + UInt32 temp_num_1s; + reader.read(&temp_num_1s); + MARISA_THROW_IF(temp_num_1s > size_, MARISA_FORMAT_ERROR); + num_1s_ = temp_num_1s; + } + ranks_.read(reader); + select0s_.read(reader); + select1s_.read(reader); + } + + void write_(Writer &writer) const { + units_.write(writer); + writer.write((UInt32)size_); + writer.write((UInt32)num_1s_); + ranks_.write(writer); + select0s_.write(writer); + select1s_.write(writer); + } + + // Disallows copy and assignment. + BitVector(const BitVector &); + BitVector &operator=(const BitVector &); +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_BIT_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/flat-vector.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/flat-vector.h new file mode 100644 index 00000000000..eeae719e27c --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/flat-vector.h @@ -0,0 +1,205 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_FLAT_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_FLAT_VECTOR_H_ + +#include "marisa/grimoire/vector/vector.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +class FlatVector { + public: +#if MARISA_WORD_SIZE == 64 + typedef UInt64 Unit; +#else // MARISA_WORD_SIZE == 64 + typedef UInt32 Unit; +#endif // MARISA_WORD_SIZE == 64 + + FlatVector() : units_(), value_size_(0), mask_(0), size_(0) {} + + void build(const Vector<UInt32> &values) { + FlatVector temp; + temp.build_(values); + swap(temp); + } + + void map(Mapper &mapper) { + FlatVector temp; + temp.map_(mapper); + swap(temp); + } + void read(Reader &reader) { + FlatVector temp; + temp.read_(reader); + swap(temp); + } + void write(Writer &writer) const { + write_(writer); + } + + UInt32 operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + + const std::size_t pos = i * value_size_; + const std::size_t unit_id = pos / MARISA_WORD_SIZE; + const std::size_t unit_offset = pos % MARISA_WORD_SIZE; + + if ((unit_offset + value_size_) <= MARISA_WORD_SIZE) { + return (UInt32)(units_[unit_id] >> unit_offset) & mask_; + } else { + return (UInt32)((units_[unit_id] >> unit_offset) + | (units_[unit_id + 1] << (MARISA_WORD_SIZE - unit_offset))) & mask_; + } + } + + std::size_t value_size() const { + return value_size_; + } + UInt32 mask() const { + return mask_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + std::size_t total_size() const { + return units_.total_size(); + } + std::size_t io_size() const { + return units_.io_size() + (sizeof(UInt32) * 2) + sizeof(UInt64); + } + + void clear() { + FlatVector().swap(*this); + } + void swap(FlatVector &rhs) { + units_.swap(rhs.units_); + marisa::swap(value_size_, rhs.value_size_); + marisa::swap(mask_, rhs.mask_); + marisa::swap(size_, rhs.size_); + } + + private: + Vector<Unit> units_; + std::size_t value_size_; + UInt32 mask_; + std::size_t size_; + + void build_(const Vector<UInt32> &values) { + UInt32 max_value = 0; + for (std::size_t i = 0; i < values.size(); ++i) { + if (values[i] > max_value) { + max_value = values[i]; + } + } + + std::size_t value_size = 0; + while (max_value != 0) { + ++value_size; + max_value >>= 1; + } + + std::size_t num_units = values.empty() ? 0 : (64 / MARISA_WORD_SIZE); + if (value_size != 0) { + num_units = (std::size_t)( + (((UInt64)value_size * values.size()) + (MARISA_WORD_SIZE - 1)) + / MARISA_WORD_SIZE); + num_units += num_units % (64 / MARISA_WORD_SIZE); + } + + units_.resize(num_units); + if (num_units > 0) { + units_.back() = 0; + } + + value_size_ = value_size; + if (value_size != 0) { + mask_ = MARISA_UINT32_MAX >> (32 - value_size); + } + size_ = values.size(); + + for (std::size_t i = 0; i < values.size(); ++i) { + set(i, values[i]); + } + } + + void map_(Mapper &mapper) { + units_.map(mapper); + { + UInt32 temp_value_size; + mapper.map(&temp_value_size); + MARISA_THROW_IF(temp_value_size > 32, MARISA_FORMAT_ERROR); + value_size_ = temp_value_size; + } + { + UInt32 temp_mask; + mapper.map(&temp_mask); + mask_ = temp_mask; + } + { + UInt64 temp_size; + mapper.map(&temp_size); + MARISA_THROW_IF(temp_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)temp_size; + } + } + + void read_(Reader &reader) { + units_.read(reader); + { + UInt32 temp_value_size; + reader.read(&temp_value_size); + MARISA_THROW_IF(temp_value_size > 32, MARISA_FORMAT_ERROR); + value_size_ = temp_value_size; + } + { + UInt32 temp_mask; + reader.read(&temp_mask); + mask_ = temp_mask; + } + { + UInt64 temp_size; + reader.read(&temp_size); + MARISA_THROW_IF(temp_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + size_ = (std::size_t)temp_size; + } + } + + void write_(Writer &writer) const { + units_.write(writer); + writer.write((UInt32)value_size_); + writer.write((UInt32)mask_); + writer.write((UInt64)size_); + } + + void set(std::size_t i, UInt32 value) { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + MARISA_DEBUG_IF(value > mask_, MARISA_RANGE_ERROR); + + const std::size_t pos = i * value_size_; + const std::size_t unit_id = pos / MARISA_WORD_SIZE; + const std::size_t unit_offset = pos % MARISA_WORD_SIZE; + + units_[unit_id] &= ~((Unit)mask_ << unit_offset); + units_[unit_id] |= (Unit)(value & mask_) << unit_offset; + if ((unit_offset + value_size_) > MARISA_WORD_SIZE) { + units_[unit_id + 1] &= + ~((Unit)mask_ >> (MARISA_WORD_SIZE - unit_offset)); + units_[unit_id + 1] |= + (Unit)(value & mask_) >> (MARISA_WORD_SIZE - unit_offset); + } + } + + // Disallows copy and assignment. + FlatVector(const FlatVector &); + FlatVector &operator=(const FlatVector &); +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_FLAT_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/pop-count.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/pop-count.h new file mode 100644 index 00000000000..47f4b5db306 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/pop-count.h @@ -0,0 +1,110 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_POP_COUNT_H_ +#define MARISA_GRIMOIRE_VECTOR_POP_COUNT_H_ + +#include "marisa/grimoire/intrin.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +#if MARISA_WORD_SIZE == 64 + +class PopCount { + public: + explicit PopCount(UInt64 x) : value_() { + x = (x & 0x5555555555555555ULL) + ((x & 0xAAAAAAAAAAAAAAAAULL) >> 1); + x = (x & 0x3333333333333333ULL) + ((x & 0xCCCCCCCCCCCCCCCCULL) >> 2); + x = (x & 0x0F0F0F0F0F0F0F0FULL) + ((x & 0xF0F0F0F0F0F0F0F0ULL) >> 4); + x *= 0x0101010101010101ULL; + value_ = x; + } + + std::size_t lo8() const { + return (std::size_t)(value_ & 0xFFU); + } + std::size_t lo16() const { + return (std::size_t)((value_ >> 8) & 0xFFU); + } + std::size_t lo24() const { + return (std::size_t)((value_ >> 16) & 0xFFU); + } + std::size_t lo32() const { + return (std::size_t)((value_ >> 24) & 0xFFU); + } + std::size_t lo40() const { + return (std::size_t)((value_ >> 32) & 0xFFU); + } + std::size_t lo48() const { + return (std::size_t)((value_ >> 40) & 0xFFU); + } + std::size_t lo56() const { + return (std::size_t)((value_ >> 48) & 0xFFU); + } + std::size_t lo64() const { + return (std::size_t)((value_ >> 56) & 0xFFU); + } + + static std::size_t count(UInt64 x) { +#if defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + #ifdef _MSC_VER + return __popcnt64(x); + #else // _MSC_VER + return static_cast<std::size_t>(_mm_popcnt_u64(x)); + #endif // _MSC_VER +#else // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + return PopCount(x).lo64(); +#endif // defined(MARISA_X64) && defined(MARISA_USE_POPCNT) + } + + private: + UInt64 value_; +}; + +#else // MARISA_WORD_SIZE == 64 + +class PopCount { + public: + explicit PopCount(UInt32 x) : value_() { + x = (x & 0x55555555U) + ((x & 0xAAAAAAAAU) >> 1); + x = (x & 0x33333333U) + ((x & 0xCCCCCCCCU) >> 2); + x = (x & 0x0F0F0F0FU) + ((x & 0xF0F0F0F0U) >> 4); + x *= 0x01010101U; + value_ = x; + } + + std::size_t lo8() const { + return value_ & 0xFFU; + } + std::size_t lo16() const { + return (value_ >> 8) & 0xFFU; + } + std::size_t lo24() const { + return (value_ >> 16) & 0xFFU; + } + std::size_t lo32() const { + return (value_ >> 24) & 0xFFU; + } + + static std::size_t count(UInt32 x) { +#ifdef MARISA_USE_POPCNT + #ifdef _MSC_VER + return __popcnt(x); + #else // _MSC_VER + return _mm_popcnt_u32(x); + #endif // _MSC_VER +#else // MARISA_USE_POPCNT + return PopCount(x).lo32(); +#endif // MARISA_USE_POPCNT + } + + private: + UInt32 value_; +}; + +#endif // MARISA_WORD_SIZE == 64 + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_POP_COUNT_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/rank-index.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/rank-index.h new file mode 100644 index 00000000000..c1ce476be2f --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/rank-index.h @@ -0,0 +1,82 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_ +#define MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_ + +#include "marisa/base.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +class RankIndex { + public: + RankIndex() : abs_(0), rel_lo_(0), rel_hi_(0) {} + + void set_abs(std::size_t value) { + MARISA_DEBUG_IF(value > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + abs_ = (UInt32)value; + } + void set_rel1(std::size_t value) { + MARISA_DEBUG_IF(value > 64, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~0x7FU) | (value & 0x7FU)); + } + void set_rel2(std::size_t value) { + MARISA_DEBUG_IF(value > 128, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~(0xFFU << 7)) | ((value & 0xFFU) << 7)); + } + void set_rel3(std::size_t value) { + MARISA_DEBUG_IF(value > 192, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~(0xFFU << 15)) | ((value & 0xFFU) << 15)); + } + void set_rel4(std::size_t value) { + MARISA_DEBUG_IF(value > 256, MARISA_RANGE_ERROR); + rel_lo_ = (UInt32)((rel_lo_ & ~(0x1FFU << 23)) | ((value & 0x1FFU) << 23)); + } + void set_rel5(std::size_t value) { + MARISA_DEBUG_IF(value > 320, MARISA_RANGE_ERROR); + rel_hi_ = (UInt32)((rel_hi_ & ~0x1FFU) | (value & 0x1FFU)); + } + void set_rel6(std::size_t value) { + MARISA_DEBUG_IF(value > 384, MARISA_RANGE_ERROR); + rel_hi_ = (UInt32)((rel_hi_ & ~(0x1FFU << 9)) | ((value & 0x1FFU) << 9)); + } + void set_rel7(std::size_t value) { + MARISA_DEBUG_IF(value > 448, MARISA_RANGE_ERROR); + rel_hi_ = (UInt32)((rel_hi_ & ~(0x1FFU << 18)) | ((value & 0x1FFU) << 18)); + } + + std::size_t abs() const { + return abs_; + } + std::size_t rel1() const { + return rel_lo_ & 0x7FU; + } + std::size_t rel2() const { + return (rel_lo_ >> 7) & 0xFFU; + } + std::size_t rel3() const { + return (rel_lo_ >> 15) & 0xFFU; + } + std::size_t rel4() const { + return (rel_lo_ >> 23) & 0x1FFU; + } + std::size_t rel5() const { + return rel_hi_ & 0x1FFU; + } + std::size_t rel6() const { + return (rel_hi_ >> 9) & 0x1FFU; + } + std::size_t rel7() const { + return (rel_hi_ >> 18) & 0x1FFU; + } + + private: + UInt32 abs_; + UInt32 rel_lo_; + UInt32 rel_hi_; +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/vector.h b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/vector.h new file mode 100644 index 00000000000..2bfccdbd173 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/grimoire/vector/vector.h @@ -0,0 +1,256 @@ +#ifndef MARISA_GRIMOIRE_VECTOR_VECTOR_H_ +#define MARISA_GRIMOIRE_VECTOR_VECTOR_H_ + +#include <new> + +#include "marisa/grimoire/io.h" + +namespace marisa { +namespace grimoire { +namespace vector { + +template <typename T> +class Vector { + public: + Vector() + : buf_(), objs_(NULL), const_objs_(NULL), + size_(0), capacity_(0), fixed_(false) {} + ~Vector() { + if (objs_ != NULL) { + for (std::size_t i = 0; i < size_; ++i) { + objs_[i].~T(); + } + } + } + + void map(Mapper &mapper) { + Vector temp; + temp.map_(mapper); + swap(temp); + } + + void read(Reader &reader) { + Vector temp; + temp.read_(reader); + swap(temp); + } + + void write(Writer &writer) const { + write_(writer); + } + + void push_back(const T &x) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == max_size(), MARISA_SIZE_ERROR); + reserve(size_ + 1); + new (&objs_[size_]) T(x); + ++size_; + } + + void pop_back() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + objs_[--size_].~T(); + } + + // resize() assumes that T's placement new does not throw an exception. + void resize(std::size_t size) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + reserve(size); + for (std::size_t i = size_; i < size; ++i) { + new (&objs_[i]) T; + } + for (std::size_t i = size; i < size_; ++i) { + objs_[i].~T(); + } + size_ = size; + } + + // resize() assumes that T's placement new does not throw an exception. + void resize(std::size_t size, const T &x) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + reserve(size); + for (std::size_t i = size_; i < size; ++i) { + new (&objs_[i]) T(x); + } + for (std::size_t i = size; i < size_; ++i) { + objs_[i].~T(); + } + size_ = size; + } + + void reserve(std::size_t capacity) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + if (capacity <= capacity_) { + return; + } + MARISA_DEBUG_IF(capacity > max_size(), MARISA_SIZE_ERROR); + std::size_t new_capacity = capacity; + if (capacity_ > (capacity / 2)) { + if (capacity_ > (max_size() / 2)) { + new_capacity = max_size(); + } else { + new_capacity = capacity_ * 2; + } + } + realloc(new_capacity); + } + + void shrink() { + MARISA_THROW_IF(fixed_, MARISA_STATE_ERROR); + if (size_ != capacity_) { + realloc(size_); + } + } + + void fix() { + MARISA_THROW_IF(fixed_, MARISA_STATE_ERROR); + fixed_ = true; + } + + const T *begin() const { + return const_objs_; + } + const T *end() const { + return const_objs_ + size_; + } + const T &operator[](std::size_t i) const { + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return const_objs_[i]; + } + const T &front() const { + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return const_objs_[0]; + } + const T &back() const { + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return const_objs_[size_ - 1]; + } + + T *begin() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + return objs_; + } + T *end() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + return objs_ + size_; + } + T &operator[](std::size_t i) { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR); + return objs_[i]; + } + T &front() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return objs_[0]; + } + T &back() { + MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR); + MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR); + return objs_[size_ - 1]; + } + + std::size_t size() const { + return size_; + } + std::size_t capacity() const { + return capacity_; + } + bool fixed() const { + return fixed_; + } + + bool empty() const { + return size_ == 0; + } + std::size_t total_size() const { + return sizeof(T) * size_; + } + std::size_t io_size() const { + return sizeof(UInt64) + ((total_size() + 7) & ~(std::size_t)0x07); + } + + void clear() { + Vector().swap(*this); + } + void swap(Vector &rhs) { + buf_.swap(rhs.buf_); + marisa::swap(objs_, rhs.objs_); + marisa::swap(const_objs_, rhs.const_objs_); + marisa::swap(size_, rhs.size_); + marisa::swap(capacity_, rhs.capacity_); + marisa::swap(fixed_, rhs.fixed_); + } + + static std::size_t max_size() { + return MARISA_SIZE_MAX / sizeof(T); + } + + private: + scoped_array<char> buf_; + T *objs_; + const T *const_objs_; + std::size_t size_; + std::size_t capacity_; + bool fixed_; + + void map_(Mapper &mapper) { + UInt64 total_size; + mapper.map(&total_size); + MARISA_THROW_IF(total_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF((total_size % sizeof(T)) != 0, MARISA_FORMAT_ERROR); + const std::size_t size = (std::size_t)(total_size / sizeof(T)); + mapper.map(&const_objs_, size); + mapper.seek((std::size_t)((8 - (total_size % 8)) % 8)); + size_ = size; + fix(); + } + void read_(Reader &reader) { + UInt64 total_size; + reader.read(&total_size); + MARISA_THROW_IF(total_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF((total_size % sizeof(T)) != 0, MARISA_FORMAT_ERROR); + const std::size_t size = (std::size_t)(total_size / sizeof(T)); + resize(size); + reader.read(objs_, size); + reader.seek((std::size_t)((8 - (total_size % 8)) % 8)); + } + void write_(Writer &writer) const { + writer.write((UInt64)total_size()); + writer.write(const_objs_, size_); + writer.seek((8 - (total_size() % 8)) % 8); + } + + // realloc() assumes that T's placement new does not throw an exception. + void realloc(std::size_t new_capacity) { + MARISA_DEBUG_IF(new_capacity > max_size(), MARISA_SIZE_ERROR); + + scoped_array<char> new_buf( + new (std::nothrow) char[sizeof(T) * new_capacity]); + MARISA_DEBUG_IF(new_buf.get() == NULL, MARISA_MEMORY_ERROR); + T *new_objs = reinterpret_cast<T *>(new_buf.get()); + + for (std::size_t i = 0; i < size_; ++i) { + new (&new_objs[i]) T(objs_[i]); + } + for (std::size_t i = 0; i < size_; ++i) { + objs_[i].~T(); + } + + buf_.swap(new_buf); + objs_ = new_objs; + const_objs_ = new_objs; + capacity_ = new_capacity; + } + + // Disallows copy and assignment. + Vector(const Vector &); + Vector &operator=(const Vector &); +}; + +} // namespace vector +} // namespace grimoire +} // namespace marisa + +#endif // MARISA_GRIMOIRE_VECTOR_VECTOR_H_ diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/keyset.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/keyset.cc new file mode 100644 index 00000000000..41354f7144d --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/keyset.cc @@ -0,0 +1,181 @@ +#include <new> + +#include "marisa/keyset.h" + +namespace marisa { + +Keyset::Keyset() + : base_blocks_(), base_blocks_size_(0), base_blocks_capacity_(0), + extra_blocks_(), extra_blocks_size_(0), extra_blocks_capacity_(0), + key_blocks_(), key_blocks_size_(0), key_blocks_capacity_(0), + ptr_(NULL), avail_(0), size_(0), total_length_(0) {} + +void Keyset::push_back(const Key &key) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + + char * const key_ptr = reserve(key.length()); + for (std::size_t i = 0; i < key.length(); ++i) { + key_ptr[i] = key[i]; + } + + Key &new_key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE]; + new_key.set_str(key_ptr, key.length()); + new_key.set_id(key.id()); + ++size_; + total_length_ += new_key.length(); +} + +void Keyset::push_back(const Key &key, char end_marker) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + + if ((size_ / KEY_BLOCK_SIZE) == key_blocks_size_) { + append_key_block(); + } + + char * const key_ptr = reserve(key.length() + 1); + for (std::size_t i = 0; i < key.length(); ++i) { + key_ptr[i] = key[i]; + } + key_ptr[key.length()] = end_marker; + + Key &new_key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE]; + new_key.set_str(key_ptr, key.length()); + new_key.set_id(key.id()); + ++size_; + total_length_ += new_key.length(); +} + +void Keyset::push_back(const char *str) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF(str == NULL, MARISA_NULL_ERROR); + + std::size_t length = 0; + while (str[length] != '\0') { + ++length; + } + push_back(str, length); +} + +void Keyset::push_back(const char *ptr, std::size_t length, float weight) { + MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR); + MARISA_THROW_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR); + MARISA_THROW_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR); + + char * const key_ptr = reserve(length); + for (std::size_t i = 0; i < length; ++i) { + key_ptr[i] = ptr[i]; + } + + Key &key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE]; + key.set_str(key_ptr, length); + key.set_weight(weight); + ++size_; + total_length_ += length; +} + +void Keyset::reset() { + base_blocks_size_ = 0; + extra_blocks_size_ = 0; + ptr_ = NULL; + avail_ = 0; + size_ = 0; + total_length_ = 0; +} + +void Keyset::clear() { + Keyset().swap(*this); +} + +void Keyset::swap(Keyset &rhs) { + base_blocks_.swap(rhs.base_blocks_); + marisa::swap(base_blocks_size_, rhs.base_blocks_size_); + marisa::swap(base_blocks_capacity_, rhs.base_blocks_capacity_); + extra_blocks_.swap(rhs.extra_blocks_); + marisa::swap(extra_blocks_size_, rhs.extra_blocks_size_); + marisa::swap(extra_blocks_capacity_, rhs.extra_blocks_capacity_); + key_blocks_.swap(rhs.key_blocks_); + marisa::swap(key_blocks_size_, rhs.key_blocks_size_); + marisa::swap(key_blocks_capacity_, rhs.key_blocks_capacity_); + marisa::swap(ptr_, rhs.ptr_); + marisa::swap(avail_, rhs.avail_); + marisa::swap(size_, rhs.size_); + marisa::swap(total_length_, rhs.total_length_); +} + +char *Keyset::reserve(std::size_t size) { + if ((size_ / KEY_BLOCK_SIZE) == key_blocks_size_) { + append_key_block(); + } + + if (size > EXTRA_BLOCK_SIZE) { + append_extra_block(size); + return extra_blocks_[extra_blocks_size_ - 1].get(); + } else { + if (size > avail_) { + append_base_block(); + } + ptr_ += size; + avail_ -= size; + return ptr_ - size; + } +} + +void Keyset::append_base_block() { + if (base_blocks_size_ == base_blocks_capacity_) { + const std::size_t new_capacity = + (base_blocks_size_ != 0) ? (base_blocks_size_ * 2) : 1; + scoped_array<scoped_array<char> > new_blocks( + new (std::nothrow) scoped_array<char>[new_capacity]); + MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR); + for (std::size_t i = 0; i < base_blocks_size_; ++i) { + base_blocks_[i].swap(new_blocks[i]); + } + base_blocks_.swap(new_blocks); + base_blocks_capacity_ = new_capacity; + } + if (base_blocks_[base_blocks_size_].get() == NULL) { + scoped_array<char> new_block(new (std::nothrow) char[BASE_BLOCK_SIZE]); + MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR); + base_blocks_[base_blocks_size_].swap(new_block); + } + ptr_ = base_blocks_[base_blocks_size_++].get(); + avail_ = BASE_BLOCK_SIZE; +} + +void Keyset::append_extra_block(std::size_t size) { + if (extra_blocks_size_ == extra_blocks_capacity_) { + const std::size_t new_capacity = + (extra_blocks_size_ != 0) ? (extra_blocks_size_ * 2) : 1; + scoped_array<scoped_array<char> > new_blocks( + new (std::nothrow) scoped_array<char>[new_capacity]); + MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR); + for (std::size_t i = 0; i < extra_blocks_size_; ++i) { + extra_blocks_[i].swap(new_blocks[i]); + } + extra_blocks_.swap(new_blocks); + extra_blocks_capacity_ = new_capacity; + } + scoped_array<char> new_block(new (std::nothrow) char[size]); + MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR); + extra_blocks_[extra_blocks_size_++].swap(new_block); +} + +void Keyset::append_key_block() { + if (key_blocks_size_ == key_blocks_capacity_) { + const std::size_t new_capacity = + (key_blocks_size_ != 0) ? (key_blocks_size_ * 2) : 1; + scoped_array<scoped_array<Key> > new_blocks( + new (std::nothrow) scoped_array<Key>[new_capacity]); + MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR); + for (std::size_t i = 0; i < key_blocks_size_; ++i) { + key_blocks_[i].swap(new_blocks[i]); + } + key_blocks_.swap(new_blocks); + key_blocks_capacity_ = new_capacity; + } + scoped_array<Key> new_block(new (std::nothrow) Key[KEY_BLOCK_SIZE]); + MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR); + key_blocks_[key_blocks_size_++].swap(new_block); +} + +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/trie.cc b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/trie.cc new file mode 100644 index 00000000000..68050011cc1 --- /dev/null +++ b/contrib/python/marisa-trie/py3/marisa-trie/lib/marisa/trie.cc @@ -0,0 +1,249 @@ +#include "marisa/stdio.h" +#include "marisa/iostream.h" +#include "marisa/trie.h" +#include "marisa/grimoire/trie.h" + +namespace marisa { + +Trie::Trie() : trie_() {} + +Trie::~Trie() {} + +void Trie::build(Keyset &keyset, int config_flags) { + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + temp->build(keyset, config_flags); + trie_.swap(temp); +} + +void Trie::mmap(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Mapper mapper; + mapper.open(filename); + temp->map(mapper); + trie_.swap(temp); +} + +void Trie::map(const void *ptr, std::size_t size) { + MARISA_THROW_IF((ptr == NULL) && (size != 0), MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Mapper mapper; + mapper.open(ptr, size); + temp->map(mapper); + trie_.swap(temp); +} + +void Trie::load(const char *filename) { + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(filename); + temp->read(reader); + trie_.swap(temp); +} + +void Trie::read(int fd) { + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp(new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(fd); + temp->read(reader); + trie_.swap(temp); +} + +void Trie::save(const char *filename) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR); + + grimoire::Writer writer; + writer.open(filename); + trie_->write(writer); +} + +void Trie::write(int fd) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR); + + grimoire::Writer writer; + writer.open(fd); + trie_->write(writer); +} + +bool Trie::lookup(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + return trie_->lookup(agent); +} + +void Trie::reverse_lookup(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + trie_->reverse_lookup(agent); +} + +bool Trie::common_prefix_search(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + return trie_->common_prefix_search(agent); +} + +bool Trie::predictive_search(Agent &agent) const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + if (!agent.has_state()) { + agent.init_state(); + } + return trie_->predictive_search(agent); +} + +std::size_t Trie::num_tries() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->num_tries(); +} + +std::size_t Trie::num_keys() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->num_keys(); +} + +std::size_t Trie::num_nodes() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->num_nodes(); +} + +TailMode Trie::tail_mode() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->tail_mode(); +} + +NodeOrder Trie::node_order() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->node_order(); +} + +bool Trie::empty() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->empty(); +} + +std::size_t Trie::size() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->size(); +} + +std::size_t Trie::total_size() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->total_size(); +} + +std::size_t Trie::io_size() const { + MARISA_THROW_IF(trie_.get() == NULL, MARISA_STATE_ERROR); + return trie_->io_size(); +} + +void Trie::clear() { + Trie().swap(*this); +} + +void Trie::swap(Trie &rhs) { + trie_.swap(rhs.trie_); +} + +} // namespace marisa + +#include <iostream> + +namespace marisa { + +class TrieIO { + public: + static void fread(std::FILE *file, Trie *trie) { + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp( + new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(file); + temp->read(reader); + trie->trie_.swap(temp); + } + static void fwrite(std::FILE *file, const Trie &trie) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + MARISA_THROW_IF(trie.trie_.get() == NULL, MARISA_STATE_ERROR); + grimoire::Writer writer; + writer.open(file); + trie.trie_->write(writer); + } + + static std::istream &read(std::istream &stream, Trie *trie) { + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + + scoped_ptr<grimoire::LoudsTrie> temp( + new (std::nothrow) grimoire::LoudsTrie); + MARISA_THROW_IF(temp.get() == NULL, MARISA_MEMORY_ERROR); + + grimoire::Reader reader; + reader.open(stream); + temp->read(reader); + trie->trie_.swap(temp); + return stream; + } + static std::ostream &write(std::ostream &stream, const Trie &trie) { + MARISA_THROW_IF(trie.trie_.get() == NULL, MARISA_STATE_ERROR); + grimoire::Writer writer; + writer.open(stream); + trie.trie_->write(writer); + return stream; + } +}; + +void fread(std::FILE *file, Trie *trie) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + TrieIO::fread(file, trie); +} + +void fwrite(std::FILE *file, const Trie &trie) { + MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR); + TrieIO::fwrite(file, trie); +} + +std::istream &read(std::istream &stream, Trie *trie) { + MARISA_THROW_IF(trie == NULL, MARISA_NULL_ERROR); + return TrieIO::read(stream, trie); +} + +std::ostream &write(std::ostream &stream, const Trie &trie) { + return TrieIO::write(stream, trie); +} + +std::istream &operator>>(std::istream &stream, Trie &trie) { + return read(stream, &trie); +} + +std::ostream &operator<<(std::ostream &stream, const Trie &trie) { + return write(stream, trie); +} + +} // namespace marisa diff --git a/contrib/python/marisa-trie/py3/patches/01-throw.patch b/contrib/python/marisa-trie/py3/patches/01-throw.patch new file mode 100644 index 00000000000..427f7929d36 --- /dev/null +++ b/contrib/python/marisa-trie/py3/patches/01-throw.patch @@ -0,0 +1,20 @@ +--- contrib/python/marisa-trie/py3/marisa-trie/include/marisa/exception.h (index) ++++ contrib/python/marisa-trie/py3/marisa-trie/include/marisa/exception.h (working tree) +@@ -19,7 +21,7 @@ class Exception : public std::exception { + Exception(const Exception &ex) + : std::exception(), filename_(ex.filename_), line_(ex.line_), + error_code_(ex.error_code_), error_message_(ex.error_message_) {} +- virtual ~Exception() throw() {} ++ virtual ~Exception() {} + + Exception &operator=(const Exception &rhs) { + filename_ = rhs.filename_; +@@ -42,7 +44,7 @@ class Exception : public std::exception { + return error_message_; + } + +- virtual const char *what() const throw() { ++ virtual const char *what() const noexcept { + return error_message_; + } + diff --git a/contrib/python/marisa-trie/py3/patches/03-fix-ya.make.patch b/contrib/python/marisa-trie/py3/patches/03-fix-ya.make.patch new file mode 100644 index 00000000000..23efd686a7b --- /dev/null +++ b/contrib/python/marisa-trie/py3/patches/03-fix-ya.make.patch @@ -0,0 +1,5 @@ +--- contrib/python/marisa-trie/py3/ya.make (index) ++++ contrib/python/marisa-trie/py3/ya.make (working tree) +@@ -32,1 +32,1 @@ SRCS( +- src/marisa_trie.pyx ++ src/marisa_trie.pyx=marisa_trie diff --git a/contrib/python/marisa-trie/py3/src/agent.pxd b/contrib/python/marisa-trie/py3/src/agent.pxd new file mode 100644 index 00000000000..bf019673c2c --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/agent.pxd @@ -0,0 +1,22 @@ +cimport query, key + +cdef extern from "<marisa/agent.h>" namespace "marisa" nogil: + cdef cppclass Agent: + Agent() except + + + query.Query &query() + key.Key &key() + + void set_query(char *str) + void set_query(char *ptr, int length) + void set_query(int key_id) + + void set_key(char *str) + void set_key(char *ptr, int length) + void set_key(int id) + + void clear() + + void init_state() + + void swap(Agent &rhs) diff --git a/contrib/python/marisa-trie/py3/src/base.pxd b/contrib/python/marisa-trie/py3/src/base.pxd new file mode 100644 index 00000000000..c434e82122c --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/base.pxd @@ -0,0 +1,63 @@ +cdef extern from "<marisa/base.h>": + + # A dictionary consists of 3 tries in default. Usually more tries make a + # dictionary space-efficient but time-inefficient. + ctypedef enum marisa_num_tries: + MARISA_MIN_NUM_TRIES + MARISA_MAX_NUM_TRIES + MARISA_DEFAULT_NUM_TRIES + + + # This library uses a cache technique to accelerate search functions. The + # following enumerated type marisa_cache_level gives a list of available cache + # size options. A larger cache enables faster search but takes a more space. + ctypedef enum marisa_cache_level: + MARISA_HUGE_CACHE + MARISA_LARGE_CACHE + MARISA_NORMAL_CACHE + MARISA_SMALL_CACHE + MARISA_TINY_CACHE + MARISA_DEFAULT_CACHE + + # This library provides 2 kinds of TAIL implementations. + ctypedef enum marisa_tail_mode: + # MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is + # available if and only if the last labels do not contain a NULL character. + # If MARISA_TEXT_TAIL is specified and a NULL character exists in the last + # labels, the setting is automatically switched to MARISA_BINARY_TAIL. + MARISA_TEXT_TAIL + + # MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses + # a bit vector to detect the end of a sequence, instead of NULL characters. + # So, MARISA_BINARY_TAIL requires a larger space if the average length of + # labels is greater than 8. + MARISA_BINARY_TAIL + + MARISA_DEFAULT_TAIL + + # The arrangement of nodes affects the time cost of matching and the order of + # predictive search. + ctypedef enum marisa_node_order: + # MARISA_LABEL_ORDER arranges nodes in ascending label order. + # MARISA_LABEL_ORDER is useful if an application needs to predict keys in + # label order. + MARISA_LABEL_ORDER + + # MARISA_WEIGHT_ORDER arranges nodes in descending weight order. + # MARISA_WEIGHT_ORDER is generally a better choice because it enables faster + # matching. + MARISA_WEIGHT_ORDER + MARISA_DEFAULT_ORDER + + ctypedef enum marisa_config_mask: + MARISA_NUM_TRIES_MASK + MARISA_CACHE_LEVEL_MASK + MARISA_TAIL_MODE_MASK + MARISA_NODE_ORDER_MASK + MARISA_CONFIG_MASK + + +cdef extern from "<marisa/base.h>" namespace "marisa": + ctypedef marisa_cache_level CacheLevel + ctypedef marisa_tail_mode TailMode + ctypedef marisa_node_order NodeOrder diff --git a/contrib/python/marisa-trie/py3/src/iostream.pxd b/contrib/python/marisa-trie/py3/src/iostream.pxd new file mode 100644 index 00000000000..435ee85bb0b --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/iostream.pxd @@ -0,0 +1,7 @@ +from std_iostream cimport istream, ostream +from trie cimport Trie + +cdef extern from "<marisa/iostream.h>" namespace "marisa" nogil: + + istream &read(istream &stream, Trie *trie) + ostream &write(ostream &stream, Trie &trie) diff --git a/contrib/python/marisa-trie/py3/src/key.pxd b/contrib/python/marisa-trie/py3/src/key.pxd new file mode 100644 index 00000000000..d99dee5e04a --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/key.pxd @@ -0,0 +1,22 @@ +cdef extern from "<marisa/key.h>" namespace "marisa" nogil: + + cdef cppclass Key: + Key() + Key(Key &query) + + #Key &operator=(Key &query) + + char operator[](int i) + + void set_str(char *str) + void set_str(char *ptr, int length) + void set_id(int id) + void set_weight(float weight) + + char *ptr() + int length() + int id() + float weight() + + void clear() + void swap(Key &rhs) diff --git a/contrib/python/marisa-trie/py3/src/keyset.pxd b/contrib/python/marisa-trie/py3/src/keyset.pxd new file mode 100644 index 00000000000..1fb99a40c55 --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/keyset.pxd @@ -0,0 +1,30 @@ +cimport key + +cdef extern from "<marisa/keyset.h>" namespace "marisa" nogil: + cdef cppclass Keyset: + +# cdef enum constants: +# BASE_BLOCK_SIZE = 4096 +# EXTRA_BLOCK_SIZE = 1024 +# KEY_BLOCK_SIZE = 256 + + Keyset() + + void push_back(key.Key &key) + void push_back(key.Key &key, char end_marker) + + void push_back(char *str) + void push_back(char *ptr, int length) + void push_back(char *ptr, int length, float weight) + + key.Key &operator[](int i) + + int num_keys() + bint empty() + + int size() + int total_length() + + void reset() + void clear() + void swap(Keyset &rhs) diff --git a/contrib/python/marisa-trie/py3/src/marisa_trie.pyx b/contrib/python/marisa-trie/py3/src/marisa_trie.pyx new file mode 100644 index 00000000000..486de1fea65 --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/marisa_trie.pyx @@ -0,0 +1,801 @@ +# cython: profile=False, embedsignature=True + +from std_iostream cimport stringstream, istream, ostream +from libc.string cimport strncmp +cimport keyset +cimport key +cimport agent +cimport trie +cimport iostream +cimport base + +from cpython.buffer cimport PyBUF_SIMPLE, Py_buffer, PyObject_CheckBuffer, PyObject_GetBuffer, PyBuffer_Release + +import itertools +import struct +import warnings + +DEFAULT_CACHE = base.MARISA_DEFAULT_CACHE +HUGE_CACHE = base.MARISA_HUGE_CACHE +LARGE_CACHE = base.MARISA_LARGE_CACHE +NORMAL_CACHE = base.MARISA_NORMAL_CACHE +SMALL_CACHE = base.MARISA_SMALL_CACHE +TINY_CACHE = base.MARISA_TINY_CACHE + +MIN_NUM_TRIES = base.MARISA_MIN_NUM_TRIES +MAX_NUM_TRIES = base.MARISA_MAX_NUM_TRIES +DEFAULT_NUM_TRIES = base.MARISA_DEFAULT_NUM_TRIES + +# MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is +# available if and only if the last labels do not contain a NULL character. +# If MARISA_TEXT_TAIL is specified and a NULL character exists in the last +# labels, the setting is automatically switched to MARISA_BINARY_TAIL. +TEXT_TAIL = base.MARISA_TEXT_TAIL + +# MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses +# a bit vector to detect the end of a sequence, instead of NULL characters. +# So, MARISA_BINARY_TAIL requires a larger space if the average length of +# labels is greater than 8. +BINARY_TAIL = base.MARISA_BINARY_TAIL +DEFAULT_TAIL = base.MARISA_DEFAULT_TAIL + + +# MARISA_LABEL_ORDER arranges nodes in ascending label order. +# MARISA_LABEL_ORDER is useful if an application needs to predict keys in +# label order. +LABEL_ORDER = base.MARISA_LABEL_ORDER + +# MARISA_WEIGHT_ORDER arranges nodes in descending weight order. +# MARISA_WEIGHT_ORDER is generally a better choice because it enables faster +# matching. +WEIGHT_ORDER = base.MARISA_WEIGHT_ORDER +DEFAULT_ORDER = base.MARISA_DEFAULT_ORDER + + +cdef inline int getbufptr(object obj, char ** ptr, Py_ssize_t * size, Py_buffer * buf): + """Get a pointer from bytes/buffer object ``obj``. + + On success, return 0, and set ``ptr``, ``size`` and ``buf``.""" + cdef int result = -1 + ptr[0] = NULL + size[0] = 0 + if PyObject_CheckBuffer(obj) == 1: # new-style Buffer interface + result = PyObject_GetBuffer(obj, buf, PyBUF_SIMPLE) + if result == 0: + ptr[0] = <char *>buf.buf + size[0] = buf.len + return result + + +cdef inline void releasebuf(Py_buffer *buf): + """Release buffer if necessary.""" + PyBuffer_Release(buf) + + +cdef class _Trie: + cdef trie.Trie* _trie + + cdef bytes _encode_key(self, key): + return key + + cdef _get_key(self, agent.Agent& ag): + return ag.key().ptr()[:ag.key().length()] + + def __init__(self, arg=None, num_tries=DEFAULT_NUM_TRIES, binary=False, + cache_size=DEFAULT_CACHE, order=DEFAULT_ORDER, weights=None): + """ + ``arg`` can be one of the following: + + * an iterable with bytes keys; + * None (if you're going to load a trie later). + + Pass a ``weights`` iterable with expected lookup frequencies + to optimize lookup and prefix search speed. + """ + + if self._trie: + return + self._trie = new trie.Trie() + + byte_keys = (self._encode_key(key) for key in (arg or [])) + + self._build( + byte_keys, + weights, + num_tries=num_tries, + binary=binary, + cache_size=cache_size, + order=order + ) + + def __dealloc__(self): + if self._trie: + del self._trie + + def _config_flags(self, num_tries=DEFAULT_NUM_TRIES, binary=False, + cache_size=DEFAULT_CACHE, order=DEFAULT_ORDER): + if not MIN_NUM_TRIES <= num_tries <= MAX_NUM_TRIES: + raise ValueError( + "num_tries (which is %d) must be between between %d and %d" % + (num_tries, MIN_NUM_TRIES, MAX_NUM_TRIES)) + + binary_flag = BINARY_TAIL if binary else TEXT_TAIL + return num_tries | binary_flag | cache_size | order + + def _build(self, byte_keys, weights=None, **options): + if weights is None: + weights = itertools.repeat(1.0) + + cdef char* data + cdef float weight + cdef keyset.Keyset *ks = new keyset.Keyset() + + try: + for key, weight in zip(byte_keys, weights): + ks.push_back(<char *>key, len(key), weight) + self._trie.build(ks[0], self._config_flags(**options)) + finally: + del ks + + def __richcmp__(self, other, int op): + if op == 2: # == + if other is self: + return True + elif not isinstance(other, _Trie): + return False + + return (<_Trie>self)._equals(other) + elif op == 3: # != + return not (self == other) + + raise TypeError("unorderable types: {0} and {1}".format( + self.__class__, other.__class__)) + + cdef bint _equals(self, _Trie other) nogil: + cdef int num_keys = self._trie.num_keys() + cdef base.NodeOrder node_order = self._trie.node_order() + if (other._trie.num_keys() != num_keys or + other._trie.node_order() != node_order): + return False + + cdef agent.Agent ag1, ag2 + ag1.set_query(b"") + ag2.set_query(b"") + cdef int i + cdef key.Key key1, key2 + for i in range(num_keys): + self._trie.predictive_search(ag1) + other._trie.predictive_search(ag2) + key1 = ag1.key() + key2 = ag2.key() + if (key1.length() != key2.length() or + strncmp(key1.ptr(), key2.ptr(), key1.length()) != 0): + return False + return True + + def __iter__(self): + return self.iterkeys() + + def __len__(self): + return self._trie.num_keys() + + def __contains__(self, key): + cdef bytes _key = self._encode_key(key) + return self._contains(_key) + + cdef bint _contains(self, bytes key): + cdef agent.Agent ag + ag.set_query(key, len(key)) + return self._trie.lookup(ag) + + def read(self, f): + """Read a trie from an open file. + + :param file f: a "real" on-disk file object. Passing a *file-like* + object would result in an error. + + .. deprecated:: 0.7.3 + + The method will be removed in version 0.8.0. Please use + :meth:`load` instead. + """ + warnings.warn("Trie.read is deprecated and will " + "be removed in marisa_trie 0.8.0. Please use " + "Trie.load instead.", DeprecationWarning) + self._trie.read(f.fileno()) + return self + + def write(self, f): + """Write a trie to an open file. + + :param file f: a "real" on-disk file object. Passing a *file-like* + object would result in an error. + + .. deprecated:: 0.7.3 + + The method will be removed in version 0.8.0. Please use + :meth:`save` instead. + """ + warnings.warn("Trie.write is deprecated and will " + "be removed in marisa_trie 0.8.0. Please use " + "Trie.save instead.", DeprecationWarning) + self._trie.write(f.fileno()) + + def save(self, path): + """Save a trie to a specified path.""" + with open(path, 'w') as f: + self._trie.write(f.fileno()) + + def load(self, path): + """Load a trie from a specified path.""" + with open(path, 'r') as f: + self._trie.read(f.fileno()) + return self + + cpdef bytes tobytes(self) except +: + """Return raw trie content as bytes.""" + cdef stringstream stream + iostream.write((<ostream *> &stream)[0], self._trie[0]) + cdef bytes res = stream.str() + return res + + cpdef frombytes(self, bytes data) except +: + """Load a trie from raw bytes generated by :meth:`tobytes`.""" + cdef stringstream* stream = new stringstream(data) + try: + iostream.read((<istream *> stream)[0], self._trie) + finally: + del stream + return self + + def __reduce__(self): + return self.__class__, (), self.tobytes() + + __setstate__ = frombytes + + def mmap(self, path): + """Memory map the content of a trie stored in a file. + + This allows to query trie without loading it fully in memory. + """ + import sys + str_path = path.encode(sys.getfilesystemencoding()) + cdef char* c_path = str_path + self._trie.mmap(c_path) + return self + + def map(self, buffer): + """Load the trie from an object exposing the buffer protocol.""" + + cdef char *ptr = NULL + cdef Py_ssize_t size = 0 + cdef Py_buffer buf + result = getbufptr(buffer, &ptr, &size, &buf) + if result != 0: + raise ValueError("Invalid buffer.") + self._trie.map(ptr, size) + releasebuf(&buf) + return self + + def iterkeys(self, prefix=None): + """ + Return an iterator over trie keys starting with a given ``prefix``. + """ + cdef agent.Agent ag + cdef bytes b_prefix = b'' + if prefix is not None: + b_prefix = self._encode_key(prefix) + ag.set_query(b_prefix, len(b_prefix)) + + while self._trie.predictive_search(ag): + yield self._get_key(ag) + + cpdef list keys(self, prefix=None): + """Return a list of trie keys starting with a given ``prefix``.""" + # non-generator inlined version of iterkeys() + cdef list res = [] + cdef bytes b_prefix = b'' + if prefix is not None: + b_prefix = self._encode_key(prefix) + cdef agent.Agent ag + ag.set_query(b_prefix, len(b_prefix)) + + while self._trie.predictive_search(ag): + res.append(self._get_key(ag)) + + return res + + def has_keys_with_prefix(self, prefix=""): + """ + Return ``True`` if any key in the trie begins with ``prefix``. + + .. deprecated:: 0.7.3 + + The method will be removed in version 0.8.0. Please use + :meth:`iterkeys` instead. + """ + warnings.warn("Trie.has_keys_with_prefix is deprecated and will " + "be removed in marisa_trie 0.8.0. Please use " + "Trie.iterkeys instead.", DeprecationWarning) + + cdef agent.Agent ag + cdef bytes b_prefix = self._encode_key(prefix) + ag.set_query(b_prefix, len(b_prefix)) + return self._trie.predictive_search(ag) + + +cdef class BinaryTrie(_Trie): + """A trie mapping bytes keys to auto-generated unique IDs.""" + + # key_id method is not in _Trie because it won't work for BytesTrie + cpdef int key_id(self, bytes key) except -1: + """Return an ID generated for a given ``key``. + + :raises KeyError: if key is not present in this trie. + """ + cdef int res = self._key_id(key, len(key)) + if res == -1: + raise KeyError(key) + return res + + cdef int _key_id(self, char* key, int len): + cdef bint res + cdef agent.Agent ag + ag.set_query(key, len) + res = self._trie.lookup(ag) + if not res: + return -1 + return ag.key().id() + + cpdef restore_key(self, int index): + """Return a key corresponding to a given ID.""" + cdef agent.Agent ag + ag.set_query(index) + try: + self._trie.reverse_lookup(ag) + except KeyError: + raise KeyError(index) + return self._get_key(ag) + + def __getitem__(self, bytes key): + return self.key_id(key) + + def get(self, bytes key, default=None): + """ + Return an ID for a given ``key`` or ``default`` if ``key`` is + not present in this trie. + """ + cdef int res + + res = self._key_id(key, len(key)) + if res == -1: + return default + return res + + def iter_prefixes(self, bytes key): + """ + Return an iterator of all prefixes of a given key. + """ + cdef agent.Agent ag + ag.set_query(key, len(key)) + + while self._trie.common_prefix_search(ag): + yield self._get_key(ag) + + def prefixes(self, bytes key): + """ + Return a list with all prefixes of a given key. + """ + # this an inlined version of ``list(self.iter_prefixes(key))`` + + cdef list res = [] + cdef agent.Agent ag + ag.set_query(key, len(key)) + + while self._trie.common_prefix_search(ag): + res.append(self._get_key(ag)) + return res + + def items(self, bytes prefix=b""): + # inlined for speed + cdef list res = [] + cdef agent.Agent ag + ag.set_query(prefix, len(prefix)) + + while self._trie.predictive_search(ag): + res.append((self._get_key(ag), ag.key().id())) + + return res + + def iteritems(self, bytes prefix=b""): + """ + Return an iterator over items that have a prefix ``prefix``. + """ + cdef agent.Agent ag + ag.set_query(prefix, len(prefix)) + + while self._trie.predictive_search(ag): + yield self._get_key(ag), ag.key().id() + + +cdef class _UnicodeKeyedTrie(_Trie): + """ + MARISA-trie wrapper for unicode keys. + """ + cdef bytes _encode_key(self, key): + return key.encode('utf8') + + cdef _get_key(self, agent.Agent& ag): + return <unicode>_Trie._get_key(self, ag).decode('utf8') + + +cdef class Trie(_UnicodeKeyedTrie): + """A trie mapping unicode keys to auto-generated unique IDs.""" + + # key_id method is not in _Trie because it won't work for BytesTrie + cpdef int key_id(self, unicode key) except -1: + """Return an ID generated for a given ``key``. + + :raises KeyError: if key is not present in this trie. + """ + cdef bytes _key = <bytes>key.encode('utf8') + cdef int res = self._key_id(_key) + if res == -1: + raise KeyError(key) + return res + + def __getitem__(self, unicode key): + return self.key_id(key) + + def get(self, key, default=None): + """ + Return an ID for a given ``key`` or ``default`` if ``key`` is + not present in this trie. + """ + cdef bytes b_key + cdef int res + + if isinstance(key, unicode): + b_key = <bytes>(<unicode>key).encode('utf8') + else: + b_key = key + + res = self._key_id(b_key) + if res == -1: + return default + return res + + cpdef restore_key(self, int index): + """Return a key corresponding to a given ID.""" + cdef agent.Agent ag + ag.set_query(index) + try: + self._trie.reverse_lookup(ag) + except KeyError: + raise KeyError(index) + return self._get_key(ag) + + cdef int _key_id(self, char* key): + cdef bint res + cdef agent.Agent ag + ag.set_query(key) + res = self._trie.lookup(ag) + if not res: + return -1 + return ag.key().id() + + def iter_prefixes(self, unicode key): + """ + Return an iterator of all prefixes of a given key. + """ + cdef bytes b_key = <bytes>key.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_key) + + while self._trie.common_prefix_search(ag): + yield self._get_key(ag) + + def prefixes(self, unicode key): + """ + Return a list with all prefixes of a given key. + """ + # this an inlined version of ``list(self.iter_prefixes(key))`` + + cdef list res = [] + cdef bytes b_key = <bytes>key.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_key) + + while self._trie.common_prefix_search(ag): + res.append(self._get_key(ag)) + return res + + def iter_prefixes_with_ids(self, unicode key): + """ + Return an iterator of (prefix, id) pairs of all prefixes of a given key. + """ + cdef bytes b_key = <bytes>key.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_key, len(b_key)) + + while self._trie.common_prefix_search(ag): + yield (self._get_key(ag), ag.key().id()) + + def iteritems(self, unicode prefix=""): + """ + Return an iterator over items that have a prefix ``prefix``. + """ + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + yield self._get_key(ag), ag.key().id() + + def items(self, unicode prefix=""): + # inlined for speed + cdef list res = [] + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + res.append((self._get_key(ag), ag.key().id())) + + return res + + +# This symbol is not allowed in utf8 so it is safe to use +# as a separator between utf8-encoded string and binary payload. +# XXX: b'\xff' value changes sort order for BytesTrie and RecordTrie. +# See https://github.com/kmike/DAWG docs for a description of a similar issue. +cdef bytes _VALUE_SEPARATOR = b'\xff' + + +cdef class BytesTrie(_UnicodeKeyedTrie): + """A trie mapping unicode keys to lists of bytes objects. + + The mapping is implemented by appending binary values to UTF8-encoded + and storing the result in MARISA-trie. + """ + cdef bytes _b_value_separator + cdef unsigned char _c_value_separator + + def __init__(self, arg=None, bytes value_separator=_VALUE_SEPARATOR, + **options): + """ + ``arg`` must be an iterable of tuples (unicode_key, bytes_payload). + """ + super(BytesTrie, self).__init__() + + self._b_value_separator = value_separator + self._c_value_separator = <unsigned char>ord(value_separator) + + byte_keys = (self._raw_key(d[0], d[1]) for d in (arg or [])) + self._build(byte_keys, **options) + + cpdef bytes _raw_key(self, unicode key, bytes payload): + return key.encode('utf8') + self._b_value_separator + payload + + cdef bint _contains(self, bytes key): + cdef agent.Agent ag + cdef bytes _key = key + self._b_value_separator + ag.set_query(_key) + return self._trie.predictive_search(ag) + + cpdef list prefixes(self, unicode key): + """ + Return a list with all prefixes of a given key. + """ + + # XXX: is there a char-walking API in libmarisa? + # This implementation is suboptimal. + + cdef agent.Agent ag + cdef list res = [] + cdef int key_len = len(key) + cdef unicode prefix + cdef bytes b_prefix + cdef int ind = 1 + + while ind <= key_len: + prefix = key[:ind] + b_prefix = <bytes>(prefix.encode('utf8') + self._b_value_separator) + ag.set_query(b_prefix) + if self._trie.predictive_search(ag): + res.append(prefix) + + ind += 1 + + return res + + def __getitem__(self, key): + cdef list res = self.get(key) + if res is None: + raise KeyError(key) + return res + + cpdef get(self, key, default=None): + """ + Return a list of payloads (as byte objects) for a given key + or ``default`` if the key is not found. + """ + cdef list res + + if isinstance(key, unicode): + res = self.get_value(<unicode>key) + else: + res = self.b_get_value(key) + + if not res: + return default + return res + + cpdef list get_value(self, unicode key): + """ + Return a list of payloads (as byte objects) for a given unicode key. + """ + cdef bytes b_key = <bytes>key.encode('utf8') + return self.b_get_value(b_key) + + cpdef list b_get_value(self, bytes key): + """ + Return a list of payloads (as byte objects) for a given utf8-encoded key. + """ + cdef list res = [] + cdef bytes value + cdef bytes b_prefix = key + self._b_value_separator + cdef int prefix_len = len(b_prefix) + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + value = ag.key().ptr()[prefix_len:ag.key().length()] + res.append(value) + + return res + + cpdef list items(self, unicode prefix=""): + # copied from iteritems for speed + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef bytes value + cdef unicode key + cdef unsigned char* raw_key + cdef list res = [] + cdef int i, value_len + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + break + + key = raw_key[:i].decode('utf8') + value = raw_key[i+1:ag.key().length()] + + res.append( + (key, value) + ) + return res + + def iteritems(self, unicode prefix=""): + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef bytes value + cdef unicode key + cdef unsigned char* raw_key + cdef int i, value_len + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + break + + key = raw_key[:i].decode('utf8') + value = raw_key[i+1:ag.key().length()] + + yield key, value + + cpdef list keys(self, prefix=""): + # copied from iterkeys for speed + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef unicode key + cdef unsigned char* raw_key + cdef list res = [] + cdef int i + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + key = raw_key[:i].decode('utf8') + res.append(key) + break + return res + + def iterkeys(self, unicode prefix=""): + cdef bytes b_prefix = <bytes>prefix.encode('utf8') + cdef unicode key + cdef unsigned char* raw_key + cdef int i + + cdef agent.Agent ag + ag.set_query(b_prefix) + + while self._trie.predictive_search(ag): + raw_key = <unsigned char*>ag.key().ptr() + + for i in range(0, ag.key().length()): + if raw_key[i] == self._c_value_separator: + yield raw_key[:i].decode('utf8') + break + + +cdef class _UnpackTrie(BytesTrie): + + def __init__(self, arg=None, **options): + keys = ((d[0], self._pack(d[1])) for d in (arg or [])) + super(_UnpackTrie, self).__init__(keys, **options) + + cdef _unpack(self, bytes value): + return value + + cdef bytes _pack(self, value): + return value + + cpdef list b_get_value(self, bytes key): + cdef list values = BytesTrie.b_get_value(self, key) + return [self._unpack(val) for val in values] + + cpdef list items(self, unicode prefix=""): + cdef list items = BytesTrie.items(self, prefix) + return [(key, self._unpack(val)) for (key, val) in items] + + def iteritems(self, unicode prefix=""): + return ((key, self._unpack(val)) for key, val in BytesTrie.iteritems(self, prefix)) + + +cdef class RecordTrie(_UnpackTrie): + """A trie mapping unicode keys to lists of data tuples. + + The data is packed using :mod:`struct` module, therefore all + tuples must be of the same format. See :mod:`struct` documentation + for available format strings. + + The mapping is implemented by appending binary values to UTF8-encoded + and storing the result in MARISA-trie. + """ + cdef _struct + cdef _fmt + + def __init__(self, fmt, arg=None, **options): + """ + ``arg`` must be an iterable of tuples (unicode_key, data_tuple). + Data tuples will be converted to bytes with + ``struct.pack(fmt, *data_tuple)``. + """ + self._fmt = fmt + self._struct = struct.Struct(str(fmt)) + super(RecordTrie, self).__init__(arg, **options) + + cdef _unpack(self, bytes value): + return self._struct.unpack(value) + + cdef bytes _pack(self, value): + return self._struct.pack(*value) + + def __reduce__(self): + return self.__class__, (self._fmt, ), self.tobytes() diff --git a/contrib/python/marisa-trie/py3/src/query.pxd b/contrib/python/marisa-trie/py3/src/query.pxd new file mode 100644 index 00000000000..a650bb89658 --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/query.pxd @@ -0,0 +1,20 @@ +cdef extern from "<marisa/query.h>" namespace "marisa" nogil: + + cdef cppclass Query: + Query() + Query(Query &query) + + #Query &operator=(Query &query) + + char operator[](int i) + + void set_str(char *str) + void set_str(char *ptr, int length) + void set_id(int id) + + char *ptr() + int length() + int id() + + void clear() + void swap(Query &rhs) diff --git a/contrib/python/marisa-trie/py3/src/std_iostream.pxd b/contrib/python/marisa-trie/py3/src/std_iostream.pxd new file mode 100644 index 00000000000..bf7d0e89aa7 --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/std_iostream.pxd @@ -0,0 +1,18 @@ +from libcpp.string cimport string + +cdef extern from "<istream>" namespace "std" nogil: + cdef cppclass istream: + istream() except + + istream& read (char* s, int n) except + + + cdef cppclass ostream: + ostream() except + + ostream& write (char* s, int n) except + + +cdef extern from "<sstream>" namespace "std" nogil: + + cdef cppclass stringstream: + stringstream() + stringstream(string s) + string str () + diff --git a/contrib/python/marisa-trie/py3/src/trie.pxd b/contrib/python/marisa-trie/py3/src/trie.pxd new file mode 100644 index 00000000000..f525caf8ad6 --- /dev/null +++ b/contrib/python/marisa-trie/py3/src/trie.pxd @@ -0,0 +1,41 @@ +cimport agent +cimport base +cimport keyset + + +cdef extern from "<marisa/trie.h>" namespace "marisa" nogil: + + cdef cppclass Trie: + Trie() + + void build(keyset.Keyset &keyset, int config_flags) except + + void build(keyset.Keyset &keyset) except + + + void mmap(char *filename) except + + void map(void *ptr, int size) except + + + void load(char *filename) except + + void read(int fd) except + + + void save(char *filename) except + + void write(int fd) except + + + bint lookup(agent.Agent &agent) except + + void reverse_lookup(agent.Agent &agent) except +KeyError + bint common_prefix_search(agent.Agent &agent) except + + bint predictive_search(agent.Agent &agent) except + + + int num_tries() except + + int num_keys() except + + int num_nodes() except + + + base.TailMode tail_mode() + base.NodeOrder node_order() + + bint empty() except + + int size() except + + int total_size() except + + int io_size() except + + + void clear() except + + void swap(Trie &rhs) except + diff --git a/contrib/python/marisa-trie/py3/ya.make b/contrib/python/marisa-trie/py3/ya.make new file mode 100644 index 00000000000..b226d84e135 --- /dev/null +++ b/contrib/python/marisa-trie/py3/ya.make @@ -0,0 +1,50 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +PROVIDES(MARISA-TRIE) + +VERSION(1.2.1) + +LICENSE(MIT) + +PEERDIR( + contrib/python/setuptools +) + +ADDINCL( + contrib/python/marisa-trie/py3/marisa-trie/include + contrib/python/marisa-trie/py3/marisa-trie/lib +) + +NO_COMPILER_WARNINGS() + +NO_LINT() + +SRCS( + marisa-trie/lib/marisa/agent.cc + marisa-trie/lib/marisa/grimoire/io/mapper.cc + marisa-trie/lib/marisa/grimoire/io/reader.cc + marisa-trie/lib/marisa/grimoire/io/writer.cc + marisa-trie/lib/marisa/grimoire/trie/louds-trie.cc + marisa-trie/lib/marisa/grimoire/trie/tail.cc + marisa-trie/lib/marisa/grimoire/vector/bit-vector.cc + marisa-trie/lib/marisa/keyset.cc + marisa-trie/lib/marisa/trie.cc +) + +PY_SRCS( + TOP_LEVEL + CYTHON_DIRECTIVE + language_level=3 + CYTHON_CPP + src/marisa_trie.pyx=marisa_trie +) + +RESOURCE_FILES( + PREFIX contrib/python/marisa-trie/py3/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/marisa-trie/ya.make b/contrib/python/marisa-trie/ya.make new file mode 100644 index 00000000000..07aad350148 --- /dev/null +++ b/contrib/python/marisa-trie/ya.make @@ -0,0 +1,22 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +VERSION(Service-proxy-version) + +IF (PYTHON2) + PEERDIR(contrib/python/marisa-trie/py2) +ELSE() + PEERDIR(contrib/python/marisa-trie/py3) +ENDIF() + +NO_LINT() + +END() + +IF (NOT OPENSOURCE) + RECURSE( + py2 + py3 + ) +ENDIF() diff --git a/contrib/python/mypy-extensions/.dist-info/METADATA b/contrib/python/mypy-extensions/.dist-info/METADATA new file mode 100644 index 00000000000..abe8c62e817 --- /dev/null +++ b/contrib/python/mypy-extensions/.dist-info/METADATA @@ -0,0 +1,29 @@ +Metadata-Version: 2.4 +Name: mypy_extensions +Version: 1.1.0 +Summary: Type system extensions for programs checked with the mypy type checker. +Author-email: The mypy developers <[email protected]> +Requires-Python: >=3.8 +Description-Content-Type: text/markdown +License-Expression: MIT +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Software Development +License-File: LICENSE +Project-URL: Homepage, https://github.com/python/mypy_extensions + +Mypy Extensions +=============== + +The `mypy_extensions` module defines extensions to the Python standard +library `typing` module that are supported by the mypy type checker and +the mypyc compiler. + diff --git a/contrib/python/mypy-extensions/.dist-info/top_level.txt b/contrib/python/mypy-extensions/.dist-info/top_level.txt new file mode 100644 index 00000000000..ee216652780 --- /dev/null +++ b/contrib/python/mypy-extensions/.dist-info/top_level.txt @@ -0,0 +1 @@ +mypy_extensions diff --git a/contrib/python/mypy-extensions/LICENSE b/contrib/python/mypy-extensions/LICENSE new file mode 100644 index 00000000000..bdb7786b232 --- /dev/null +++ b/contrib/python/mypy-extensions/LICENSE @@ -0,0 +1,27 @@ +Mypy extensions are licensed under the terms of the MIT license, reproduced below. + += = = = = + +The MIT License + +Copyright (c) 2016-2017 Jukka Lehtosalo and contributors + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + += = = = = diff --git a/contrib/python/mypy-extensions/README.md b/contrib/python/mypy-extensions/README.md new file mode 100644 index 00000000000..4b280a9b34e --- /dev/null +++ b/contrib/python/mypy-extensions/README.md @@ -0,0 +1,6 @@ +Mypy Extensions +=============== + +The `mypy_extensions` module defines extensions to the Python standard +library `typing` module that are supported by the mypy type checker and +the mypyc compiler. diff --git a/contrib/python/mypy-extensions/mypy_extensions.py b/contrib/python/mypy-extensions/mypy_extensions.py new file mode 100644 index 00000000000..1910000e711 --- /dev/null +++ b/contrib/python/mypy-extensions/mypy_extensions.py @@ -0,0 +1,251 @@ +"""Defines experimental extensions to the standard "typing" module that are +supported by the mypy typechecker. + +Example usage: + from mypy_extensions import TypedDict +""" + +from typing import Any, Dict + +import sys +# _type_check is NOT a part of public typing API, it is used here only to mimic +# the (convenient) behavior of types provided by typing module. +from typing import _type_check # type: ignore + + +def _check_fails(cls, other): + try: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools', 'typing']: + # Typed dicts are only for static structural subtyping. + raise TypeError('TypedDict does not support instance and class checks') + except (AttributeError, ValueError): + pass + return False + + +def _dict_new(cls, *args, **kwargs): + return dict(*args, **kwargs) + + +def _typeddict_new(cls, _typename, _fields=None, **kwargs): + total = kwargs.pop('total', True) + if _fields is None: + _fields = kwargs + elif kwargs: + raise TypeError("TypedDict takes either a dict or keyword arguments," + " but not both") + + ns = {'__annotations__': dict(_fields), '__total__': total} + try: + # Setting correct module is necessary to make typed dict classes pickleable. + ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + + return _TypedDictMeta(_typename, (), ns, _from_functional_call=True) + + +class _TypedDictMeta(type): + def __new__(cls, name, bases, ns, total=True, _from_functional_call=False): + # Create new typed dict class object. + # This method is called directly when TypedDict is subclassed, + # or via _typeddict_new when TypedDict is instantiated. This way + # TypedDict supports all three syntaxes described in its docstring. + # Subclasses and instances of TypedDict return actual dictionaries + # via _dict_new. + + # We need the `if TypedDict in globals()` check, + # or we emit a DeprecationWarning when creating mypy_extensions.TypedDict itself + if 'TypedDict' in globals(): + import warnings + warnings.warn( + ( + "mypy_extensions.TypedDict is deprecated, " + "and will be removed in a future version. " + "Use typing.TypedDict or typing_extensions.TypedDict instead." + ), + DeprecationWarning, + stacklevel=(3 if _from_functional_call else 2) + ) + + ns['__new__'] = _typeddict_new if name == 'TypedDict' else _dict_new + tp_dict = super(_TypedDictMeta, cls).__new__(cls, name, (dict,), ns) + + anns = ns.get('__annotations__', {}) + msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" + anns = {n: _type_check(tp, msg) for n, tp in anns.items()} + for base in bases: + anns.update(base.__dict__.get('__annotations__', {})) + tp_dict.__annotations__ = anns + if not hasattr(tp_dict, '__total__'): + tp_dict.__total__ = total + return tp_dict + + __instancecheck__ = __subclasscheck__ = _check_fails + + +TypedDict = _TypedDictMeta('TypedDict', (dict,), {}) +TypedDict.__module__ = __name__ +TypedDict.__doc__ = \ + """A simple typed name space. At runtime it is equivalent to a plain dict. + + TypedDict creates a dictionary type that expects all of its + instances to have a certain set of keys, with each key + associated with a value of a consistent type. This expectation + is not checked at runtime but is only enforced by typecheckers. + Usage:: + + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + The type info could be accessed via Point2D.__annotations__. TypedDict + supports two additional equivalent forms:: + + Point2D = TypedDict('Point2D', x=int, y=int, label=str) + + class Point2D(TypedDict): + x: int + y: int + label: str + + The latter syntax is only supported in Python 3.6+, while two other + syntax forms work for 3.2+ + """ + +# Argument constructors for making more-detailed Callables. These all just +# return their type argument, to make them complete noops in terms of the +# `typing` module. + + +def Arg(type=Any, name=None): + """A normal positional argument""" + return type + + +def DefaultArg(type=Any, name=None): + """A positional argument with a default value""" + return type + + +def NamedArg(type=Any, name=None): + """A keyword-only argument""" + return type + + +def DefaultNamedArg(type=Any, name=None): + """A keyword-only argument with a default value""" + return type + + +def VarArg(type=Any): + """A *args-style variadic positional argument""" + return type + + +def KwArg(type=Any): + """A **kwargs-style variadic keyword argument""" + return type + + +# Return type that indicates a function does not return +# Deprecated, use typing or typing_extensions variants instead +class _DEPRECATED_NoReturn: pass + + +def trait(cls): + return cls + + +def mypyc_attr(*attrs, **kwattrs): + return lambda x: x + + +# TODO: We may want to try to properly apply this to any type +# variables left over... +class _FlexibleAliasClsApplied: + def __init__(self, val): + self.val = val + + def __getitem__(self, args): + return self.val + + +class _FlexibleAliasCls: + def __getitem__(self, args): + return _FlexibleAliasClsApplied(args[-1]) + + +FlexibleAlias = _FlexibleAliasCls() + + +class _NativeIntMeta(type): + def __instancecheck__(cls, inst): + return isinstance(inst, int) + + +_sentinel = object() + + +class i64(metaclass=_NativeIntMeta): + def __new__(cls, x=0, base=_sentinel): + if base is not _sentinel: + return int(x, base) + return int(x) + + +class i32(metaclass=_NativeIntMeta): + def __new__(cls, x=0, base=_sentinel): + if base is not _sentinel: + return int(x, base) + return int(x) + + +class i16(metaclass=_NativeIntMeta): + def __new__(cls, x=0, base=_sentinel): + if base is not _sentinel: + return int(x, base) + return int(x) + + +class u8(metaclass=_NativeIntMeta): + def __new__(cls, x=0, base=_sentinel): + if base is not _sentinel: + return int(x, base) + return int(x) + + +for _int_type in i64, i32, i16, u8: + _int_type.__doc__ = \ + """A native fixed-width integer type when used with mypyc. + + In code not compiled with mypyc, behaves like the 'int' type in these + runtime contexts: + + * {name}(x[, base=n]) converts a number or string to 'int' + * isinstance(x, {name}) is the same as isinstance(x, int) + """.format(name=_int_type.__name__) +del _int_type + + +def _warn_deprecation(name: str, module_globals: Dict[str, Any]) -> Any: + if (val := module_globals.get(f"_DEPRECATED_{name}")) is None: + msg = f"module '{__name__}' has no attribute '{name}'" + raise AttributeError(msg) + module_globals[name] = val + if name in {"NoReturn"}: + msg = ( + f"'mypy_extensions.{name}' is deprecated, " + "and will be removed in a future version. " + f"Use 'typing.{name}' or 'typing_extensions.{name}' instead" + ) + else: + assert False, f"Add deprecation message for 'mypy_extensions.{name}'" + import warnings + warnings.warn(msg, DeprecationWarning, stacklevel=3) + return val + + +def __getattr__(name: str) -> Any: + return _warn_deprecation(name, module_globals=globals()) diff --git a/contrib/python/mypy-extensions/ya.make b/contrib/python/mypy-extensions/ya.make new file mode 100644 index 00000000000..f1156d20bb9 --- /dev/null +++ b/contrib/python/mypy-extensions/ya.make @@ -0,0 +1,22 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(1.1.0) + +LICENSE(MIT) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + mypy_extensions.py +) + +RESOURCE_FILES( + PREFIX contrib/python/mypy-extensions/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/pathspec/.dist-info/METADATA b/contrib/python/pathspec/.dist-info/METADATA new file mode 100644 index 00000000000..9cef72483c0 --- /dev/null +++ b/contrib/python/pathspec/.dist-info/METADATA @@ -0,0 +1,647 @@ +Metadata-Version: 2.1 +Name: pathspec +Version: 0.12.1 +Summary: Utility library for gitignore style pattern matching of file paths. +Author-email: "Caleb P. Burns" <[email protected]> +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Utilities +Project-URL: Documentation, https://python-path-specification.readthedocs.io/en/latest/index.html +Project-URL: Issue Tracker, https://github.com/cpburnz/python-pathspec/issues +Project-URL: Source Code, https://github.com/cpburnz/python-pathspec + + +PathSpec +======== + +*pathspec* is a utility library for pattern matching of file paths. So +far this only includes Git's wildmatch pattern matching which itself is +derived from Rsync's wildmatch. Git uses wildmatch for its `gitignore`_ +files. + +.. _`gitignore`: http://git-scm.com/docs/gitignore + + +Tutorial +-------- + +Say you have a "Projects" directory and you want to back it up, but only +certain files, and ignore others depending on certain conditions:: + + >>> import pathspec + >>> # The gitignore-style patterns for files to select, but we're including + >>> # instead of ignoring. + >>> spec_text = """ + ... + ... # This is a comment because the line begins with a hash: "#" + ... + ... # Include several project directories (and all descendants) relative to + ... # the current directory. To reference a directory you must end with a + ... # slash: "/" + ... /project-a/ + ... /project-b/ + ... /project-c/ + ... + ... # Patterns can be negated by prefixing with exclamation mark: "!" + ... + ... # Ignore temporary files beginning or ending with "~" and ending with + ... # ".swp". + ... !~* + ... !*~ + ... !*.swp + ... + ... # These are python projects so ignore compiled python files from + ... # testing. + ... !*.pyc + ... + ... # Ignore the build directories but only directly under the project + ... # directories. + ... !/*/build/ + ... + ... """ + +We want to use the ``GitWildMatchPattern`` class to compile our patterns. The +``PathSpec`` class provides an interface around pattern implementations:: + + >>> spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, spec_text.splitlines()) + +That may be a mouthful but it allows for additional patterns to be implemented +in the future without them having to deal with anything but matching the paths +sent to them. ``GitWildMatchPattern`` is the implementation of the actual +pattern which internally gets converted into a regular expression. ``PathSpec`` +is a simple wrapper around a list of compiled patterns. + +To make things simpler, we can use the registered name for a pattern class +instead of always having to provide a reference to the class itself. The +``GitWildMatchPattern`` class is registered as **gitwildmatch**:: + + >>> spec = pathspec.PathSpec.from_lines('gitwildmatch', spec_text.splitlines()) + +If we wanted to manually compile the patterns we can just do the following:: + + >>> patterns = map(pathspec.patterns.GitWildMatchPattern, spec_text.splitlines()) + >>> spec = PathSpec(patterns) + +``PathSpec.from_lines()`` is simply a class method which does just that. + +If you want to load the patterns from file, you can pass the file instance +directly as well:: + + >>> with open('patterns.list', 'r') as fh: + >>> spec = pathspec.PathSpec.from_lines('gitwildmatch', fh) + +You can perform matching on a whole directory tree with:: + + >>> matches = spec.match_tree('path/to/directory') + +Or you can perform matching on a specific set of file paths with:: + + >>> matches = spec.match_files(file_paths) + +Or check to see if an individual file matches:: + + >>> is_matched = spec.match_file(file_path) + +There is a specialized class, ``pathspec.GitIgnoreSpec``, which more closely +implements the behavior of **gitignore**. This uses ``GitWildMatchPattern`` +pattern by default and handles some edge cases differently from the generic +``PathSpec`` class. ``GitIgnoreSpec`` can be used without specifying the pattern +factory:: + + >>> spec = pathspec.GitIgnoreSpec.from_lines(spec_text.splitlines()) + + +License +------- + +*pathspec* is licensed under the `Mozilla Public License Version 2.0`_. See +`LICENSE`_ or the `FAQ`_ for more information. + +In summary, you may use *pathspec* with any closed or open source project +without affecting the license of the larger work so long as you: + +- give credit where credit is due, + +- and release any custom changes made to *pathspec*. + +.. _`Mozilla Public License Version 2.0`: http://www.mozilla.org/MPL/2.0 +.. _`LICENSE`: LICENSE +.. _`FAQ`: http://www.mozilla.org/MPL/2.0/FAQ.html + + +Source +------ + +The source code for *pathspec* is available from the GitHub repo +`cpburnz/python-pathspec`_. + +.. _`cpburnz/python-pathspec`: https://github.com/cpburnz/python-pathspec + + +Installation +------------ + +*pathspec* is available for install through `PyPI`_:: + + pip install pathspec + +*pathspec* can also be built from source. The following packages will be +required: + +- `build`_ (>=0.6.0) + +*pathspec* can then be built and installed with:: + + python -m build + pip install dist/pathspec-*-py3-none-any.whl + +.. _`PyPI`: http://pypi.python.org/pypi/pathspec +.. _`build`: https://pypi.org/project/build/ + + +Documentation +------------- + +Documentation for *pathspec* is available on `Read the Docs`_. + +.. _`Read the Docs`: https://python-path-specification.readthedocs.io + + +Other Languages +--------------- + +The related project `pathspec-ruby`_ (by *highb*) provides a similar library as +a `Ruby gem`_. + +.. _`pathspec-ruby`: https://github.com/highb/pathspec-ruby +.. _`Ruby gem`: https://rubygems.org/gems/pathspec + + + +Change History +============== + + +0.12.1 (2023-12-10) +------------------- + +Bug fixes: + +- `Issue #84`_: PathSpec.match_file() returns None since 0.12.0. + + +.. _`Issue #84`: https://github.com/cpburnz/python-pathspec/issues/84 + + +0.12.0 (2023-12-09) +------------------- + +Major changes: + +- Dropped support of EOL Python 3.7. See `Pull #82`_. + + +API changes: + +- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` (with a leading underscore) has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. + +New features: + +- Added `pathspec.pathspec.PathSpec.check_*()` methods. These methods behave similarly to `.match_*()` but return additional information in the `pathspec.util.CheckResult` objects (e.g., `CheckResult.index` indicates the index of the last pattern that matched the file). +- Added `pathspec.pattern.RegexPattern.pattern` attribute which stores the original, uncompiled pattern. + +Bug fixes: + +- `Issue #81`_: GitIgnoreSpec behaviors differ from git. +- `Pull #83`_: Fix ReadTheDocs builds. + +Improvements: + +- Mark Python 3.12 as supported. See `Pull #82`_. +- Improve test debugging. +- Improve type hint on *on_error* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Improve type hint on *on_error* parameter on `pathspec.util.iter_tree_entries()`. + + +.. _`Issue #81`: https://github.com/cpburnz/python-pathspec/issues/81 +.. _`Pull #82`: https://github.com/cpburnz/python-pathspec/pull/82 +.. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 + + +0.11.2 (2023-07-28) +------------------- + +New features: + +- `Issue #80`_: match_files with negated path spec. `pathspec.PathSpec.match_*()` now have a `negate` parameter to make using *.gitignore* logic easier and more efficient. + +Bug fixes: + +- `Pull #76`_: Add edge case: patterns that end with an escaped space +- `Issue #77`_/`Pull #78`_: Negate with caret symbol as with the exclamation mark. + + +.. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 +.. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 +.. _`Pull #78`: https://github.com/cpburnz/python-pathspec/pull/78/ +.. _`Issue #80`: https://github.com/cpburnz/python-pathspec/issues/80 + + +0.11.1 (2023-03-14) +------------------- + +Bug fixes: + +- `Issue #74`_: Include directory should override exclude file. + +Improvements: + +- `Pull #75`_: Fix partially unknown PathLike type. +- Convert `os.PathLike` to a string properly using `os.fspath`. + + +.. _`Issue #74`: https://github.com/cpburnz/python-pathspec/issues/74 +.. _`Pull #75`: https://github.com/cpburnz/python-pathspec/pull/75 + + +0.11.0 (2023-01-24) +------------------- + +Major changes: + +- Changed build backend to `flit_core.buildapi`_ from `setuptools.build_meta`_. Building with `setuptools` through `setup.py` is still supported for distributions that need it. See `Issue #72`_. + +Improvements: + +- `Issue #72`_/`Pull #73`_: Please consider switching the build-system to flit_core to ease setuptools bootstrap. + + +.. _`flit_core.buildapi`: https://flit.pypa.io/en/latest/index.html +.. _`Issue #72`: https://github.com/cpburnz/python-pathspec/issues/72 +.. _`Pull #73`: https://github.com/cpburnz/python-pathspec/pull/73 + + +0.10.3 (2022-12-09) +------------------- + +New features: + +- Added utility function `pathspec.util.append_dir_sep()` to aid in distinguishing between directories and files on the file-system. See `Issue #65`_. + +Bug fixes: + +- `Issue #66`_/`Pull #67`_: Package not marked as py.typed. +- `Issue #68`_: Exports are considered private. +- `Issue #70`_/`Pull #71`_: 'Self' string literal type is Unknown in pyright. + +Improvements: + +- `Issue #65`_: Checking directories via match_file() does not work on Path objects. + + +.. _`Issue #65`: https://github.com/cpburnz/python-pathspec/issues/65 +.. _`Issue #66`: https://github.com/cpburnz/python-pathspec/issues/66 +.. _`Pull #67`: https://github.com/cpburnz/python-pathspec/pull/67 +.. _`Issue #68`: https://github.com/cpburnz/python-pathspec/issues/68 +.. _`Issue #70`: https://github.com/cpburnz/python-pathspec/issues/70 +.. _`Pull #71`: https://github.com/cpburnz/python-pathspec/pull/71 + + +0.10.2 (2022-11-12) +------------------- + +Bug fixes: + +- Fix failing tests on Windows. +- Type hint on *root* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Type hint on *root* parameter on `pathspec.pathspec.PathSpec.match_tree_files()`. +- Type hint on *root* parameter on `pathspec.util.iter_tree_entries()`. +- Type hint on *root* parameter on `pathspec.util.iter_tree_files()`. +- `Issue #64`_: IndexError with my .gitignore file when trying to build a Python package. + +Improvements: + +- `Pull #58`_: CI: add GitHub Actions test workflow. + + +.. _`Pull #58`: https://github.com/cpburnz/python-pathspec/pull/58 +.. _`Issue #64`: https://github.com/cpburnz/python-pathspec/issues/64 + + +0.10.1 (2022-09-02) +------------------- + +Bug fixes: + +- Fix documentation on `pathspec.pattern.RegexPattern.match_file()`. +- `Pull #60`_: Remove redundant wheel dep from pyproject.toml. +- `Issue #61`_: Dist failure for Fedora, CentOS, EPEL. +- `Issue #62`_: Since version 0.10.0 pure wildcard does not work in some cases. + +Improvements: + +- Restore support for legacy installations using `setup.py`. See `Issue #61`_. + + +.. _`Pull #60`: https://github.com/cpburnz/python-pathspec/pull/60 +.. _`Issue #61`: https://github.com/cpburnz/python-pathspec/issues/61 +.. _`Issue #62`: https://github.com/cpburnz/python-pathspec/issues/62 + + +0.10.0 (2022-08-30) +------------------- + +Major changes: + +- Dropped support of EOL Python 2.7, 3.5, 3.6. See `Issue #47`_. +- The *gitwildmatch* pattern `dir/*` is now handled the same as `dir/`. This means `dir/*` will now match all descendants rather than only direct children. See `Issue #19`_. +- Added `pathspec.GitIgnoreSpec` class (see new features). +- Changed build system to `pyproject.toml`_ and build backend to `setuptools.build_meta`_ which may have unforeseen consequences. +- Renamed GitHub project from `python-path-specification`_ to `python-pathspec`_. See `Issue #35`_. + +API changes: + +- Deprecated: `pathspec.util.match_files()` is an old function no longer used. +- Deprecated: `pathspec.match_files()` is an old function no longer used. +- Deprecated: `pathspec.util.normalize_files()` is no longer used. +- Deprecated: `pathspec.util.iter_tree()` is an alias for `pathspec.util.iter_tree_files()`. +- Deprecated: `pathspec.iter_tree()` is an alias for `pathspec.util.iter_tree_files()`. +- Deprecated: `pathspec.pattern.Pattern.match()` is no longer used. Use or implement + `pathspec.pattern.Pattern.match_file()`. + +New features: + +- Added class `pathspec.gitignore.GitIgnoreSpec` (with alias `pathspec.GitIgnoreSpec`) to implement *gitignore* behavior not possible with standard `PathSpec` class. The particular *gitignore* behavior implemented is prioritizing patterns matching the file directly over matching an ancestor directory. + +Bug fixes: + +- `Issue #19`_: Files inside an ignored sub-directory are not matched. +- `Issue #41`_: Incorrectly (?) matches files inside directories that do match. +- `Pull #51`_: Refactor deprecated unittest aliases for Python 3.11 compatibility. +- `Issue #53`_: Symlink pathspec_meta.py breaks Windows. +- `Issue #54`_: test_util.py uses os.symlink which can fail on Windows. +- `Issue #55`_: Backslashes at start of pattern not handled correctly. +- `Pull #56`_: pyproject.toml: include subpackages in setuptools config +- `Issue #57`_: `!` doesn't exclude files in directories if the pattern doesn't have a trailing slash. + +Improvements: + +- Support Python 3.10, 3.11. +- Modernize code to Python 3.7. +- `Issue #52`_: match_files() is not a pure generator function, and it impacts tree_*() gravely. + + +.. _`python-path-specification`: https://github.com/cpburnz/python-path-specification +.. _`python-pathspec`: https://github.com/cpburnz/python-pathspec +.. _`pyproject.toml`: https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/ +.. _`setuptools.build_meta`: https://setuptools.pypa.io/en/latest/build_meta.html +.. _`Issue #19`: https://github.com/cpburnz/python-pathspec/issues/19 +.. _`Issue #35`: https://github.com/cpburnz/python-pathspec/issues/35 +.. _`Issue #41`: https://github.com/cpburnz/python-pathspec/issues/41 +.. _`Issue #47`: https://github.com/cpburnz/python-pathspec/issues/47 +.. _`Pull #51`: https://github.com/cpburnz/python-pathspec/pull/51 +.. _`Issue #52`: https://github.com/cpburnz/python-pathspec/issues/52 +.. _`Issue #53`: https://github.com/cpburnz/python-pathspec/issues/53 +.. _`Issue #54`: https://github.com/cpburnz/python-pathspec/issues/54 +.. _`Issue #55`: https://github.com/cpburnz/python-pathspec/issues/55 +.. _`Pull #56`: https://github.com/cpburnz/python-pathspec/pull/56 +.. _`Issue #57`: https://github.com/cpburnz/python-pathspec/issues/57 + + +0.9.0 (2021-07-17) +------------------ + +- `Issue #44`_/`Pull #50`_: Raise `GitWildMatchPatternError` for invalid git patterns. +- `Pull #45`_: Fix for duplicate leading double-asterisk, and edge cases. +- `Issue #46`_: Fix matching absolute paths. +- API change: `util.normalize_files()` now returns a `Dict[str, List[pathlike]]` instead of a `Dict[str, pathlike]`. +- Added type hinting. + +.. _`Issue #44`: https://github.com/cpburnz/python-pathspec/issues/44 +.. _`Pull #45`: https://github.com/cpburnz/python-pathspec/pull/45 +.. _`Issue #46`: https://github.com/cpburnz/python-pathspec/issues/46 +.. _`Pull #50`: https://github.com/cpburnz/python-pathspec/pull/50 + + +0.8.1 (2020-11-07) +------------------ + +- `Pull #43`_: Add support for addition operator. + +.. _`Pull #43`: https://github.com/cpburnz/python-pathspec/pull/43 + + +0.8.0 (2020-04-09) +------------------ + +- `Issue #30`_: Expose what patterns matched paths. Added `util.detailed_match_files()`. +- `Issue #31`_: `match_tree()` doesn't return symlinks. +- `Issue #34`_: Support `pathlib.Path`\ s. +- Add `PathSpec.match_tree_entries` and `util.iter_tree_entries()` to support directories and symlinks. +- API change: `match_tree()` has been renamed to `match_tree_files()`. The old name `match_tree()` is still available as an alias. +- API change: `match_tree_files()` now returns symlinks. This is a bug fix but it will change the returned results. + +.. _`Issue #30`: https://github.com/cpburnz/python-pathspec/issues/30 +.. _`Issue #31`: https://github.com/cpburnz/python-pathspec/issues/31 +.. _`Issue #34`: https://github.com/cpburnz/python-pathspec/issues/34 + + +0.7.0 (2019-12-27) +------------------ + +- `Pull #28`_: Add support for Python 3.8, and drop Python 3.4. +- `Pull #29`_: Publish bdist wheel. + +.. _`Pull #28`: https://github.com/cpburnz/python-pathspec/pull/28 +.. _`Pull #29`: https://github.com/cpburnz/python-pathspec/pull/29 + + +0.6.0 (2019-10-03) +------------------ + +- `Pull #24`_: Drop support for Python 2.6, 3.2, and 3.3. +- `Pull #25`_: Update README.rst. +- `Pull #26`_: Method to escape gitwildmatch. + +.. _`Pull #24`: https://github.com/cpburnz/python-pathspec/pull/24 +.. _`Pull #25`: https://github.com/cpburnz/python-pathspec/pull/25 +.. _`Pull #26`: https://github.com/cpburnz/python-pathspec/pull/26 + + +0.5.9 (2018-09-15) +------------------ + +- Fixed file system error handling. + + +0.5.8 (2018-09-15) +------------------ + +- Improved type checking. +- Created scripts to test Python 2.6 because Tox removed support for it. +- Improved byte string handling in Python 3. +- `Issue #22`_: Handle dangling symlinks. + +.. _`Issue #22`: https://github.com/cpburnz/python-pathspec/issues/22 + + +0.5.7 (2018-08-14) +------------------ + +- `Issue #21`_: Fix collections deprecation warning. + +.. _`Issue #21`: https://github.com/cpburnz/python-pathspec/issues/21 + + +0.5.6 (2018-04-06) +------------------ + +- Improved unit tests. +- Improved type checking. +- `Issue #20`_: Support current directory prefix. + +.. _`Issue #20`: https://github.com/cpburnz/python-pathspec/issues/20 + + +0.5.5 (2017-09-09) +------------------ + +- Add documentation link to README. + + +0.5.4 (2017-09-09) +------------------ + +- `Pull #17`_: Add link to Ruby implementation of *pathspec*. +- Add sphinx documentation. + +.. _`Pull #17`: https://github.com/cpburnz/python-pathspec/pull/17 + + +0.5.3 (2017-07-01) +------------------ + +- `Issue #14`_: Fix byte strings for Python 3. +- `Pull #15`_: Include "LICENSE" in source package. +- `Issue #16`_: Support Python 2.6. + +.. _`Issue #14`: https://github.com/cpburnz/python-pathspec/issues/14 +.. _`Pull #15`: https://github.com/cpburnz/python-pathspec/pull/15 +.. _`Issue #16`: https://github.com/cpburnz/python-pathspec/issues/16 + + +0.5.2 (2017-04-04) +------------------ + +- Fixed change log. + + +0.5.1 (2017-04-04) +------------------ + +- `Pull #13`_: Add equality methods to `PathSpec` and `RegexPattern`. + +.. _`Pull #13`: https://github.com/cpburnz/python-pathspec/pull/13 + + +0.5.0 (2016-08-22) +------------------ + +- `Issue #12`_: Add `PathSpec.match_file()`. +- Renamed `gitignore.GitIgnorePattern` to `patterns.gitwildmatch.GitWildMatchPattern`. +- Deprecated `gitignore.GitIgnorePattern`. + +.. _`Issue #12`: https://github.com/cpburnz/python-pathspec/issues/12 + + +0.4.0 (2016-07-15) +------------------ + +- `Issue #11`_: Support converting patterns into regular expressions without compiling them. +- API change: Subclasses of `RegexPattern` should implement `pattern_to_regex()`. + +.. _`Issue #11`: https://github.com/cpburnz/python-pathspec/issues/11 + + +0.3.4 (2015-08-24) +------------------ + +- `Pull #7`_: Fixed non-recursive links. +- `Pull #8`_: Fixed edge cases in gitignore patterns. +- `Pull #9`_: Fixed minor usage documentation. +- Fixed recursion detection. +- Fixed trivial incompatibility with Python 3.2. + +.. _`Pull #7`: https://github.com/cpburnz/python-pathspec/pull/7 +.. _`Pull #8`: https://github.com/cpburnz/python-pathspec/pull/8 +.. _`Pull #9`: https://github.com/cpburnz/python-pathspec/pull/9 + + +0.3.3 (2014-11-21) +------------------ + +- Improved documentation. + + +0.3.2 (2014-11-08) +------------------ + +- `Pull #5`_: Use tox for testing. +- `Issue #6`_: Fixed matching Windows paths. +- Improved documentation. +- API change: `spec.match_tree()` and `spec.match_files()` now return iterators instead of sets. + +.. _`Pull #5`: https://github.com/cpburnz/python-pathspec/pull/5 +.. _`Issue #6`: https://github.com/cpburnz/python-pathspec/issues/6 + + +0.3.1 (2014-09-17) +------------------ + +- Updated README. + + +0.3.0 (2014-09-17) +------------------ + +- `Pull #3`_: Fixed trailing slash in gitignore patterns. +- `Pull #4`_: Fixed test for trailing slash in gitignore patterns. +- Added registered patterns. + +.. _`Pull #3`: https://github.com/cpburnz/python-pathspec/pull/3 +.. _`Pull #4`: https://github.com/cpburnz/python-pathspec/pull/4 + + +0.2.2 (2013-12-17) +------------------ + +- Fixed setup.py. + + +0.2.1 (2013-12-17) +------------------ + +- Added tests. +- Fixed comment gitignore patterns. +- Fixed relative path gitignore patterns. + + +0.2.0 (2013-12-07) +------------------ + +- Initial release. + diff --git a/contrib/python/pathspec/.dist-info/top_level.txt b/contrib/python/pathspec/.dist-info/top_level.txt new file mode 100644 index 00000000000..6486958df08 --- /dev/null +++ b/contrib/python/pathspec/.dist-info/top_level.txt @@ -0,0 +1 @@ +pathspec diff --git a/contrib/python/pathspec/LICENSE b/contrib/python/pathspec/LICENSE new file mode 100644 index 00000000000..14e2f777f6c --- /dev/null +++ b/contrib/python/pathspec/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/contrib/python/pathspec/README-dist.rst b/contrib/python/pathspec/README-dist.rst new file mode 100644 index 00000000000..e7752fcc97e --- /dev/null +++ b/contrib/python/pathspec/README-dist.rst @@ -0,0 +1,620 @@ + +PathSpec +======== + +*pathspec* is a utility library for pattern matching of file paths. So +far this only includes Git's wildmatch pattern matching which itself is +derived from Rsync's wildmatch. Git uses wildmatch for its `gitignore`_ +files. + +.. _`gitignore`: http://git-scm.com/docs/gitignore + + +Tutorial +-------- + +Say you have a "Projects" directory and you want to back it up, but only +certain files, and ignore others depending on certain conditions:: + + >>> import pathspec + >>> # The gitignore-style patterns for files to select, but we're including + >>> # instead of ignoring. + >>> spec_text = """ + ... + ... # This is a comment because the line begins with a hash: "#" + ... + ... # Include several project directories (and all descendants) relative to + ... # the current directory. To reference a directory you must end with a + ... # slash: "/" + ... /project-a/ + ... /project-b/ + ... /project-c/ + ... + ... # Patterns can be negated by prefixing with exclamation mark: "!" + ... + ... # Ignore temporary files beginning or ending with "~" and ending with + ... # ".swp". + ... !~* + ... !*~ + ... !*.swp + ... + ... # These are python projects so ignore compiled python files from + ... # testing. + ... !*.pyc + ... + ... # Ignore the build directories but only directly under the project + ... # directories. + ... !/*/build/ + ... + ... """ + +We want to use the ``GitWildMatchPattern`` class to compile our patterns. The +``PathSpec`` class provides an interface around pattern implementations:: + + >>> spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, spec_text.splitlines()) + +That may be a mouthful but it allows for additional patterns to be implemented +in the future without them having to deal with anything but matching the paths +sent to them. ``GitWildMatchPattern`` is the implementation of the actual +pattern which internally gets converted into a regular expression. ``PathSpec`` +is a simple wrapper around a list of compiled patterns. + +To make things simpler, we can use the registered name for a pattern class +instead of always having to provide a reference to the class itself. The +``GitWildMatchPattern`` class is registered as **gitwildmatch**:: + + >>> spec = pathspec.PathSpec.from_lines('gitwildmatch', spec_text.splitlines()) + +If we wanted to manually compile the patterns we can just do the following:: + + >>> patterns = map(pathspec.patterns.GitWildMatchPattern, spec_text.splitlines()) + >>> spec = PathSpec(patterns) + +``PathSpec.from_lines()`` is simply a class method which does just that. + +If you want to load the patterns from file, you can pass the file instance +directly as well:: + + >>> with open('patterns.list', 'r') as fh: + >>> spec = pathspec.PathSpec.from_lines('gitwildmatch', fh) + +You can perform matching on a whole directory tree with:: + + >>> matches = spec.match_tree('path/to/directory') + +Or you can perform matching on a specific set of file paths with:: + + >>> matches = spec.match_files(file_paths) + +Or check to see if an individual file matches:: + + >>> is_matched = spec.match_file(file_path) + +There is a specialized class, ``pathspec.GitIgnoreSpec``, which more closely +implements the behavior of **gitignore**. This uses ``GitWildMatchPattern`` +pattern by default and handles some edge cases differently from the generic +``PathSpec`` class. ``GitIgnoreSpec`` can be used without specifying the pattern +factory:: + + >>> spec = pathspec.GitIgnoreSpec.from_lines(spec_text.splitlines()) + + +License +------- + +*pathspec* is licensed under the `Mozilla Public License Version 2.0`_. See +`LICENSE`_ or the `FAQ`_ for more information. + +In summary, you may use *pathspec* with any closed or open source project +without affecting the license of the larger work so long as you: + +- give credit where credit is due, + +- and release any custom changes made to *pathspec*. + +.. _`Mozilla Public License Version 2.0`: http://www.mozilla.org/MPL/2.0 +.. _`LICENSE`: LICENSE +.. _`FAQ`: http://www.mozilla.org/MPL/2.0/FAQ.html + + +Source +------ + +The source code for *pathspec* is available from the GitHub repo +`cpburnz/python-pathspec`_. + +.. _`cpburnz/python-pathspec`: https://github.com/cpburnz/python-pathspec + + +Installation +------------ + +*pathspec* is available for install through `PyPI`_:: + + pip install pathspec + +*pathspec* can also be built from source. The following packages will be +required: + +- `build`_ (>=0.6.0) + +*pathspec* can then be built and installed with:: + + python -m build + pip install dist/pathspec-*-py3-none-any.whl + +.. _`PyPI`: http://pypi.python.org/pypi/pathspec +.. _`build`: https://pypi.org/project/build/ + + +Documentation +------------- + +Documentation for *pathspec* is available on `Read the Docs`_. + +.. _`Read the Docs`: https://python-path-specification.readthedocs.io + + +Other Languages +--------------- + +The related project `pathspec-ruby`_ (by *highb*) provides a similar library as +a `Ruby gem`_. + +.. _`pathspec-ruby`: https://github.com/highb/pathspec-ruby +.. _`Ruby gem`: https://rubygems.org/gems/pathspec + + + +Change History +============== + + +0.12.1 (2023-12-10) +------------------- + +Bug fixes: + +- `Issue #84`_: PathSpec.match_file() returns None since 0.12.0. + + +.. _`Issue #84`: https://github.com/cpburnz/python-pathspec/issues/84 + + +0.12.0 (2023-12-09) +------------------- + +Major changes: + +- Dropped support of EOL Python 3.7. See `Pull #82`_. + + +API changes: + +- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` (with a leading underscore) has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. + +New features: + +- Added `pathspec.pathspec.PathSpec.check_*()` methods. These methods behave similarly to `.match_*()` but return additional information in the `pathspec.util.CheckResult` objects (e.g., `CheckResult.index` indicates the index of the last pattern that matched the file). +- Added `pathspec.pattern.RegexPattern.pattern` attribute which stores the original, uncompiled pattern. + +Bug fixes: + +- `Issue #81`_: GitIgnoreSpec behaviors differ from git. +- `Pull #83`_: Fix ReadTheDocs builds. + +Improvements: + +- Mark Python 3.12 as supported. See `Pull #82`_. +- Improve test debugging. +- Improve type hint on *on_error* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Improve type hint on *on_error* parameter on `pathspec.util.iter_tree_entries()`. + + +.. _`Issue #81`: https://github.com/cpburnz/python-pathspec/issues/81 +.. _`Pull #82`: https://github.com/cpburnz/python-pathspec/pull/82 +.. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 + + +0.11.2 (2023-07-28) +------------------- + +New features: + +- `Issue #80`_: match_files with negated path spec. `pathspec.PathSpec.match_*()` now have a `negate` parameter to make using *.gitignore* logic easier and more efficient. + +Bug fixes: + +- `Pull #76`_: Add edge case: patterns that end with an escaped space +- `Issue #77`_/`Pull #78`_: Negate with caret symbol as with the exclamation mark. + + +.. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 +.. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 +.. _`Pull #78`: https://github.com/cpburnz/python-pathspec/pull/78/ +.. _`Issue #80`: https://github.com/cpburnz/python-pathspec/issues/80 + + +0.11.1 (2023-03-14) +------------------- + +Bug fixes: + +- `Issue #74`_: Include directory should override exclude file. + +Improvements: + +- `Pull #75`_: Fix partially unknown PathLike type. +- Convert `os.PathLike` to a string properly using `os.fspath`. + + +.. _`Issue #74`: https://github.com/cpburnz/python-pathspec/issues/74 +.. _`Pull #75`: https://github.com/cpburnz/python-pathspec/pull/75 + + +0.11.0 (2023-01-24) +------------------- + +Major changes: + +- Changed build backend to `flit_core.buildapi`_ from `setuptools.build_meta`_. Building with `setuptools` through `setup.py` is still supported for distributions that need it. See `Issue #72`_. + +Improvements: + +- `Issue #72`_/`Pull #73`_: Please consider switching the build-system to flit_core to ease setuptools bootstrap. + + +.. _`flit_core.buildapi`: https://flit.pypa.io/en/latest/index.html +.. _`Issue #72`: https://github.com/cpburnz/python-pathspec/issues/72 +.. _`Pull #73`: https://github.com/cpburnz/python-pathspec/pull/73 + + +0.10.3 (2022-12-09) +------------------- + +New features: + +- Added utility function `pathspec.util.append_dir_sep()` to aid in distinguishing between directories and files on the file-system. See `Issue #65`_. + +Bug fixes: + +- `Issue #66`_/`Pull #67`_: Package not marked as py.typed. +- `Issue #68`_: Exports are considered private. +- `Issue #70`_/`Pull #71`_: 'Self' string literal type is Unknown in pyright. + +Improvements: + +- `Issue #65`_: Checking directories via match_file() does not work on Path objects. + + +.. _`Issue #65`: https://github.com/cpburnz/python-pathspec/issues/65 +.. _`Issue #66`: https://github.com/cpburnz/python-pathspec/issues/66 +.. _`Pull #67`: https://github.com/cpburnz/python-pathspec/pull/67 +.. _`Issue #68`: https://github.com/cpburnz/python-pathspec/issues/68 +.. _`Issue #70`: https://github.com/cpburnz/python-pathspec/issues/70 +.. _`Pull #71`: https://github.com/cpburnz/python-pathspec/pull/71 + + +0.10.2 (2022-11-12) +------------------- + +Bug fixes: + +- Fix failing tests on Windows. +- Type hint on *root* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Type hint on *root* parameter on `pathspec.pathspec.PathSpec.match_tree_files()`. +- Type hint on *root* parameter on `pathspec.util.iter_tree_entries()`. +- Type hint on *root* parameter on `pathspec.util.iter_tree_files()`. +- `Issue #64`_: IndexError with my .gitignore file when trying to build a Python package. + +Improvements: + +- `Pull #58`_: CI: add GitHub Actions test workflow. + + +.. _`Pull #58`: https://github.com/cpburnz/python-pathspec/pull/58 +.. _`Issue #64`: https://github.com/cpburnz/python-pathspec/issues/64 + + +0.10.1 (2022-09-02) +------------------- + +Bug fixes: + +- Fix documentation on `pathspec.pattern.RegexPattern.match_file()`. +- `Pull #60`_: Remove redundant wheel dep from pyproject.toml. +- `Issue #61`_: Dist failure for Fedora, CentOS, EPEL. +- `Issue #62`_: Since version 0.10.0 pure wildcard does not work in some cases. + +Improvements: + +- Restore support for legacy installations using `setup.py`. See `Issue #61`_. + + +.. _`Pull #60`: https://github.com/cpburnz/python-pathspec/pull/60 +.. _`Issue #61`: https://github.com/cpburnz/python-pathspec/issues/61 +.. _`Issue #62`: https://github.com/cpburnz/python-pathspec/issues/62 + + +0.10.0 (2022-08-30) +------------------- + +Major changes: + +- Dropped support of EOL Python 2.7, 3.5, 3.6. See `Issue #47`_. +- The *gitwildmatch* pattern `dir/*` is now handled the same as `dir/`. This means `dir/*` will now match all descendants rather than only direct children. See `Issue #19`_. +- Added `pathspec.GitIgnoreSpec` class (see new features). +- Changed build system to `pyproject.toml`_ and build backend to `setuptools.build_meta`_ which may have unforeseen consequences. +- Renamed GitHub project from `python-path-specification`_ to `python-pathspec`_. See `Issue #35`_. + +API changes: + +- Deprecated: `pathspec.util.match_files()` is an old function no longer used. +- Deprecated: `pathspec.match_files()` is an old function no longer used. +- Deprecated: `pathspec.util.normalize_files()` is no longer used. +- Deprecated: `pathspec.util.iter_tree()` is an alias for `pathspec.util.iter_tree_files()`. +- Deprecated: `pathspec.iter_tree()` is an alias for `pathspec.util.iter_tree_files()`. +- Deprecated: `pathspec.pattern.Pattern.match()` is no longer used. Use or implement + `pathspec.pattern.Pattern.match_file()`. + +New features: + +- Added class `pathspec.gitignore.GitIgnoreSpec` (with alias `pathspec.GitIgnoreSpec`) to implement *gitignore* behavior not possible with standard `PathSpec` class. The particular *gitignore* behavior implemented is prioritizing patterns matching the file directly over matching an ancestor directory. + +Bug fixes: + +- `Issue #19`_: Files inside an ignored sub-directory are not matched. +- `Issue #41`_: Incorrectly (?) matches files inside directories that do match. +- `Pull #51`_: Refactor deprecated unittest aliases for Python 3.11 compatibility. +- `Issue #53`_: Symlink pathspec_meta.py breaks Windows. +- `Issue #54`_: test_util.py uses os.symlink which can fail on Windows. +- `Issue #55`_: Backslashes at start of pattern not handled correctly. +- `Pull #56`_: pyproject.toml: include subpackages in setuptools config +- `Issue #57`_: `!` doesn't exclude files in directories if the pattern doesn't have a trailing slash. + +Improvements: + +- Support Python 3.10, 3.11. +- Modernize code to Python 3.7. +- `Issue #52`_: match_files() is not a pure generator function, and it impacts tree_*() gravely. + + +.. _`python-path-specification`: https://github.com/cpburnz/python-path-specification +.. _`python-pathspec`: https://github.com/cpburnz/python-pathspec +.. _`pyproject.toml`: https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/ +.. _`setuptools.build_meta`: https://setuptools.pypa.io/en/latest/build_meta.html +.. _`Issue #19`: https://github.com/cpburnz/python-pathspec/issues/19 +.. _`Issue #35`: https://github.com/cpburnz/python-pathspec/issues/35 +.. _`Issue #41`: https://github.com/cpburnz/python-pathspec/issues/41 +.. _`Issue #47`: https://github.com/cpburnz/python-pathspec/issues/47 +.. _`Pull #51`: https://github.com/cpburnz/python-pathspec/pull/51 +.. _`Issue #52`: https://github.com/cpburnz/python-pathspec/issues/52 +.. _`Issue #53`: https://github.com/cpburnz/python-pathspec/issues/53 +.. _`Issue #54`: https://github.com/cpburnz/python-pathspec/issues/54 +.. _`Issue #55`: https://github.com/cpburnz/python-pathspec/issues/55 +.. _`Pull #56`: https://github.com/cpburnz/python-pathspec/pull/56 +.. _`Issue #57`: https://github.com/cpburnz/python-pathspec/issues/57 + + +0.9.0 (2021-07-17) +------------------ + +- `Issue #44`_/`Pull #50`_: Raise `GitWildMatchPatternError` for invalid git patterns. +- `Pull #45`_: Fix for duplicate leading double-asterisk, and edge cases. +- `Issue #46`_: Fix matching absolute paths. +- API change: `util.normalize_files()` now returns a `Dict[str, List[pathlike]]` instead of a `Dict[str, pathlike]`. +- Added type hinting. + +.. _`Issue #44`: https://github.com/cpburnz/python-pathspec/issues/44 +.. _`Pull #45`: https://github.com/cpburnz/python-pathspec/pull/45 +.. _`Issue #46`: https://github.com/cpburnz/python-pathspec/issues/46 +.. _`Pull #50`: https://github.com/cpburnz/python-pathspec/pull/50 + + +0.8.1 (2020-11-07) +------------------ + +- `Pull #43`_: Add support for addition operator. + +.. _`Pull #43`: https://github.com/cpburnz/python-pathspec/pull/43 + + +0.8.0 (2020-04-09) +------------------ + +- `Issue #30`_: Expose what patterns matched paths. Added `util.detailed_match_files()`. +- `Issue #31`_: `match_tree()` doesn't return symlinks. +- `Issue #34`_: Support `pathlib.Path`\ s. +- Add `PathSpec.match_tree_entries` and `util.iter_tree_entries()` to support directories and symlinks. +- API change: `match_tree()` has been renamed to `match_tree_files()`. The old name `match_tree()` is still available as an alias. +- API change: `match_tree_files()` now returns symlinks. This is a bug fix but it will change the returned results. + +.. _`Issue #30`: https://github.com/cpburnz/python-pathspec/issues/30 +.. _`Issue #31`: https://github.com/cpburnz/python-pathspec/issues/31 +.. _`Issue #34`: https://github.com/cpburnz/python-pathspec/issues/34 + + +0.7.0 (2019-12-27) +------------------ + +- `Pull #28`_: Add support for Python 3.8, and drop Python 3.4. +- `Pull #29`_: Publish bdist wheel. + +.. _`Pull #28`: https://github.com/cpburnz/python-pathspec/pull/28 +.. _`Pull #29`: https://github.com/cpburnz/python-pathspec/pull/29 + + +0.6.0 (2019-10-03) +------------------ + +- `Pull #24`_: Drop support for Python 2.6, 3.2, and 3.3. +- `Pull #25`_: Update README.rst. +- `Pull #26`_: Method to escape gitwildmatch. + +.. _`Pull #24`: https://github.com/cpburnz/python-pathspec/pull/24 +.. _`Pull #25`: https://github.com/cpburnz/python-pathspec/pull/25 +.. _`Pull #26`: https://github.com/cpburnz/python-pathspec/pull/26 + + +0.5.9 (2018-09-15) +------------------ + +- Fixed file system error handling. + + +0.5.8 (2018-09-15) +------------------ + +- Improved type checking. +- Created scripts to test Python 2.6 because Tox removed support for it. +- Improved byte string handling in Python 3. +- `Issue #22`_: Handle dangling symlinks. + +.. _`Issue #22`: https://github.com/cpburnz/python-pathspec/issues/22 + + +0.5.7 (2018-08-14) +------------------ + +- `Issue #21`_: Fix collections deprecation warning. + +.. _`Issue #21`: https://github.com/cpburnz/python-pathspec/issues/21 + + +0.5.6 (2018-04-06) +------------------ + +- Improved unit tests. +- Improved type checking. +- `Issue #20`_: Support current directory prefix. + +.. _`Issue #20`: https://github.com/cpburnz/python-pathspec/issues/20 + + +0.5.5 (2017-09-09) +------------------ + +- Add documentation link to README. + + +0.5.4 (2017-09-09) +------------------ + +- `Pull #17`_: Add link to Ruby implementation of *pathspec*. +- Add sphinx documentation. + +.. _`Pull #17`: https://github.com/cpburnz/python-pathspec/pull/17 + + +0.5.3 (2017-07-01) +------------------ + +- `Issue #14`_: Fix byte strings for Python 3. +- `Pull #15`_: Include "LICENSE" in source package. +- `Issue #16`_: Support Python 2.6. + +.. _`Issue #14`: https://github.com/cpburnz/python-pathspec/issues/14 +.. _`Pull #15`: https://github.com/cpburnz/python-pathspec/pull/15 +.. _`Issue #16`: https://github.com/cpburnz/python-pathspec/issues/16 + + +0.5.2 (2017-04-04) +------------------ + +- Fixed change log. + + +0.5.1 (2017-04-04) +------------------ + +- `Pull #13`_: Add equality methods to `PathSpec` and `RegexPattern`. + +.. _`Pull #13`: https://github.com/cpburnz/python-pathspec/pull/13 + + +0.5.0 (2016-08-22) +------------------ + +- `Issue #12`_: Add `PathSpec.match_file()`. +- Renamed `gitignore.GitIgnorePattern` to `patterns.gitwildmatch.GitWildMatchPattern`. +- Deprecated `gitignore.GitIgnorePattern`. + +.. _`Issue #12`: https://github.com/cpburnz/python-pathspec/issues/12 + + +0.4.0 (2016-07-15) +------------------ + +- `Issue #11`_: Support converting patterns into regular expressions without compiling them. +- API change: Subclasses of `RegexPattern` should implement `pattern_to_regex()`. + +.. _`Issue #11`: https://github.com/cpburnz/python-pathspec/issues/11 + + +0.3.4 (2015-08-24) +------------------ + +- `Pull #7`_: Fixed non-recursive links. +- `Pull #8`_: Fixed edge cases in gitignore patterns. +- `Pull #9`_: Fixed minor usage documentation. +- Fixed recursion detection. +- Fixed trivial incompatibility with Python 3.2. + +.. _`Pull #7`: https://github.com/cpburnz/python-pathspec/pull/7 +.. _`Pull #8`: https://github.com/cpburnz/python-pathspec/pull/8 +.. _`Pull #9`: https://github.com/cpburnz/python-pathspec/pull/9 + + +0.3.3 (2014-11-21) +------------------ + +- Improved documentation. + + +0.3.2 (2014-11-08) +------------------ + +- `Pull #5`_: Use tox for testing. +- `Issue #6`_: Fixed matching Windows paths. +- Improved documentation. +- API change: `spec.match_tree()` and `spec.match_files()` now return iterators instead of sets. + +.. _`Pull #5`: https://github.com/cpburnz/python-pathspec/pull/5 +.. _`Issue #6`: https://github.com/cpburnz/python-pathspec/issues/6 + + +0.3.1 (2014-09-17) +------------------ + +- Updated README. + + +0.3.0 (2014-09-17) +------------------ + +- `Pull #3`_: Fixed trailing slash in gitignore patterns. +- `Pull #4`_: Fixed test for trailing slash in gitignore patterns. +- Added registered patterns. + +.. _`Pull #3`: https://github.com/cpburnz/python-pathspec/pull/3 +.. _`Pull #4`: https://github.com/cpburnz/python-pathspec/pull/4 + + +0.2.2 (2013-12-17) +------------------ + +- Fixed setup.py. + + +0.2.1 (2013-12-17) +------------------ + +- Added tests. +- Fixed comment gitignore patterns. +- Fixed relative path gitignore patterns. + + +0.2.0 (2013-12-07) +------------------ + +- Initial release. diff --git a/contrib/python/pathspec/README.rst b/contrib/python/pathspec/README.rst new file mode 100644 index 00000000000..e934d401fa2 --- /dev/null +++ b/contrib/python/pathspec/README.rst @@ -0,0 +1,165 @@ + +PathSpec +======== + +*pathspec* is a utility library for pattern matching of file paths. So +far this only includes Git's wildmatch pattern matching which itself is +derived from Rsync's wildmatch. Git uses wildmatch for its `gitignore`_ +files. + +.. _`gitignore`: http://git-scm.com/docs/gitignore + + +Tutorial +-------- + +Say you have a "Projects" directory and you want to back it up, but only +certain files, and ignore others depending on certain conditions:: + + >>> import pathspec + >>> # The gitignore-style patterns for files to select, but we're including + >>> # instead of ignoring. + >>> spec_text = """ + ... + ... # This is a comment because the line begins with a hash: "#" + ... + ... # Include several project directories (and all descendants) relative to + ... # the current directory. To reference a directory you must end with a + ... # slash: "/" + ... /project-a/ + ... /project-b/ + ... /project-c/ + ... + ... # Patterns can be negated by prefixing with exclamation mark: "!" + ... + ... # Ignore temporary files beginning or ending with "~" and ending with + ... # ".swp". + ... !~* + ... !*~ + ... !*.swp + ... + ... # These are python projects so ignore compiled python files from + ... # testing. + ... !*.pyc + ... + ... # Ignore the build directories but only directly under the project + ... # directories. + ... !/*/build/ + ... + ... """ + +We want to use the ``GitWildMatchPattern`` class to compile our patterns. The +``PathSpec`` class provides an interface around pattern implementations:: + + >>> spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, spec_text.splitlines()) + +That may be a mouthful but it allows for additional patterns to be implemented +in the future without them having to deal with anything but matching the paths +sent to them. ``GitWildMatchPattern`` is the implementation of the actual +pattern which internally gets converted into a regular expression. ``PathSpec`` +is a simple wrapper around a list of compiled patterns. + +To make things simpler, we can use the registered name for a pattern class +instead of always having to provide a reference to the class itself. The +``GitWildMatchPattern`` class is registered as **gitwildmatch**:: + + >>> spec = pathspec.PathSpec.from_lines('gitwildmatch', spec_text.splitlines()) + +If we wanted to manually compile the patterns we can just do the following:: + + >>> patterns = map(pathspec.patterns.GitWildMatchPattern, spec_text.splitlines()) + >>> spec = PathSpec(patterns) + +``PathSpec.from_lines()`` is simply a class method which does just that. + +If you want to load the patterns from file, you can pass the file instance +directly as well:: + + >>> with open('patterns.list', 'r') as fh: + >>> spec = pathspec.PathSpec.from_lines('gitwildmatch', fh) + +You can perform matching on a whole directory tree with:: + + >>> matches = spec.match_tree('path/to/directory') + +Or you can perform matching on a specific set of file paths with:: + + >>> matches = spec.match_files(file_paths) + +Or check to see if an individual file matches:: + + >>> is_matched = spec.match_file(file_path) + +There is a specialized class, ``pathspec.GitIgnoreSpec``, which more closely +implements the behavior of **gitignore**. This uses ``GitWildMatchPattern`` +pattern by default and handles some edge cases differently from the generic +``PathSpec`` class. ``GitIgnoreSpec`` can be used without specifying the pattern +factory:: + + >>> spec = pathspec.GitIgnoreSpec.from_lines(spec_text.splitlines()) + + +License +------- + +*pathspec* is licensed under the `Mozilla Public License Version 2.0`_. See +`LICENSE`_ or the `FAQ`_ for more information. + +In summary, you may use *pathspec* with any closed or open source project +without affecting the license of the larger work so long as you: + +- give credit where credit is due, + +- and release any custom changes made to *pathspec*. + +.. _`Mozilla Public License Version 2.0`: http://www.mozilla.org/MPL/2.0 +.. _`LICENSE`: LICENSE +.. _`FAQ`: http://www.mozilla.org/MPL/2.0/FAQ.html + + +Source +------ + +The source code for *pathspec* is available from the GitHub repo +`cpburnz/python-pathspec`_. + +.. _`cpburnz/python-pathspec`: https://github.com/cpburnz/python-pathspec + + +Installation +------------ + +*pathspec* is available for install through `PyPI`_:: + + pip install pathspec + +*pathspec* can also be built from source. The following packages will be +required: + +- `build`_ (>=0.6.0) + +*pathspec* can then be built and installed with:: + + python -m build + pip install dist/pathspec-*-py3-none-any.whl + +.. _`PyPI`: http://pypi.python.org/pypi/pathspec +.. _`build`: https://pypi.org/project/build/ + + +Documentation +------------- + +Documentation for *pathspec* is available on `Read the Docs`_. + +.. _`Read the Docs`: https://python-path-specification.readthedocs.io + + +Other Languages +--------------- + +The related project `pathspec-ruby`_ (by *highb*) provides a similar library as +a `Ruby gem`_. + +.. _`pathspec-ruby`: https://github.com/highb/pathspec-ruby +.. _`Ruby gem`: https://rubygems.org/gems/pathspec diff --git a/contrib/python/pathspec/pathspec/__init__.py b/contrib/python/pathspec/pathspec/__init__.py new file mode 100644 index 00000000000..32e03f75f93 --- /dev/null +++ b/contrib/python/pathspec/pathspec/__init__.py @@ -0,0 +1,76 @@ +""" +The *pathspec* package provides pattern matching for file paths. So far +this only includes Git's wildmatch pattern matching (the style used for +".gitignore" files). + +The following classes are imported and made available from the root of +the `pathspec` package: + +- :class:`pathspec.gitignore.GitIgnoreSpec` + +- :class:`pathspec.pathspec.PathSpec` + +- :class:`pathspec.pattern.Pattern` + +- :class:`pathspec.pattern.RegexPattern` + +- :class:`pathspec.util.RecursionError` + +The following functions are also imported: + +- :func:`pathspec.util.lookup_pattern` + +The following deprecated functions are also imported to maintain +backward compatibility: + +- :func:`pathspec.util.iter_tree` which is an alias for + :func:`pathspec.util.iter_tree_files`. + +- :func:`pathspec.util.match_files` +""" + +from .gitignore import ( + GitIgnoreSpec) +from .pathspec import ( + PathSpec) +from .pattern import ( + Pattern, + RegexPattern) +from .util import ( + RecursionError, + iter_tree, + lookup_pattern, + match_files) + +from ._meta import ( + __author__, + __copyright__, + __credits__, + __license__, + __version__, +) + +# Load pattern implementations. +from . import patterns + +# DEPRECATED: Expose the `GitIgnorePattern` class in the root module for +# backward compatibility with v0.4. +from .patterns.gitwildmatch import GitIgnorePattern + +# Declare private imports as part of the public interface. Deprecated +# imports are deliberately excluded. +__all__ = [ + 'GitIgnoreSpec', + 'PathSpec', + 'Pattern', + 'RecursionError', + 'RegexPattern', + '__author__', + '__copyright__', + '__credits__', + '__license__', + '__version__', + 'iter_tree', + 'lookup_pattern', + 'match_files', +] diff --git a/contrib/python/pathspec/pathspec/_meta.py b/contrib/python/pathspec/pathspec/_meta.py new file mode 100644 index 00000000000..4d8c89d4467 --- /dev/null +++ b/contrib/python/pathspec/pathspec/_meta.py @@ -0,0 +1,58 @@ +""" +This module contains the project meta-data. +""" + +__author__ = "Caleb P. Burns" +__copyright__ = "Copyright © 2013-2023 Caleb P. Burns" +__credits__ = [ + "dahlia <https://github.com/dahlia>", + "highb <https://github.com/highb>", + "029xue <https://github.com/029xue>", + "mikexstudios <https://github.com/mikexstudios>", + "nhumrich <https://github.com/nhumrich>", + "davidfraser <https://github.com/davidfraser>", + "demurgos <https://github.com/demurgos>", + "ghickman <https://github.com/ghickman>", + "nvie <https://github.com/nvie>", + "adrienverge <https://github.com/adrienverge>", + "AndersBlomdell <https://github.com/AndersBlomdell>", + "thmxv <https://github.com/thmxv>", + "wimglenn <https://github.com/wimglenn>", + "hugovk <https://github.com/hugovk>", + "dcecile <https://github.com/dcecile>", + "mroutis <https://github.com/mroutis>", + "jdufresne <https://github.com/jdufresne>", + "groodt <https://github.com/groodt>", + "ftrofin <https://github.com/ftrofin>", + "pykong <https://github.com/pykong>", + "nhhollander <https://github.com/nhhollander>", + "KOLANICH <https://github.com/KOLANICH>", + "JonjonHays <https://github.com/JonjonHays>", + "Isaac0616 <https://github.com/Isaac0616>", + "SebastiaanZ <https://github.com/SebastiaanZ>", + "RoelAdriaans <https://github.com/RoelAdriaans>", + "raviselker <https://github.com/raviselker>", + "johanvergeer <https://github.com/johanvergeer>", + "danjer <https://github.com/danjer>", + "jhbuhrman <https://github.com/jhbuhrman>", + "WPDOrdina <https://github.com/WPDOrdina>", + "tirkarthi <https://github.com/tirkarthi>", + "jayvdb <https://github.com/jayvdb>", + "jwodder <https://github.com/jwodder>", + "kloczek <https://github.com/kloczek>", + "orens <https://github.com/orens>", + "spMohanty <https://github.com/spMohanty>", + "ichard26 <https://github.com/ichard26>", + "jack1142 <https://github.com/jack1142>", + "mgorny <https://github.com/mgorny>", + "bzakdd <https://github.com/bzakdd>", + "haimat <https://github.com/haimat>", + "Avasam <https://github.com/Avasam>", + "yschroeder <https://github.com/yschroeder>", + "axesider <https://github.com/axesider>", + "tomruk <https://github.com/tomruk>", + "oprypin <https://github.com/oprypin>", + "kurtmckee <https://github.com/kurtmckee>", +] +__license__ = "MPL 2.0" +__version__ = "0.12.1" diff --git a/contrib/python/pathspec/pathspec/gitignore.py b/contrib/python/pathspec/pathspec/gitignore.py new file mode 100644 index 00000000000..994a2c74a1e --- /dev/null +++ b/contrib/python/pathspec/pathspec/gitignore.py @@ -0,0 +1,157 @@ +""" +This module provides :class:`.GitIgnoreSpec` which replicates +*.gitignore* behavior. +""" + +from typing import ( + AnyStr, + Callable, # Replaced by `collections.abc.Callable` in 3.9. + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Tuple, # Replaced by `tuple` in 3.9. + Type, # Replaced by `type` in 3.9. + TypeVar, + Union, # Replaced by `X | Y` in 3.10. + cast, + overload) + +from .pathspec import ( + PathSpec) +from .pattern import ( + Pattern) +from .patterns.gitwildmatch import ( + GitWildMatchPattern, + _DIR_MARK) +from .util import ( + _is_iterable) + +Self = TypeVar("Self", bound="GitIgnoreSpec") +""" +:class:`GitIgnoreSpec` self type hint to support Python v<3.11 using PEP +673 recommendation. +""" + + +class GitIgnoreSpec(PathSpec): + """ + The :class:`GitIgnoreSpec` class extends :class:`pathspec.pathspec.PathSpec` to + replicate *.gitignore* behavior. + """ + + def __eq__(self, other: object) -> bool: + """ + Tests the equality of this gitignore-spec with *other* (:class:`GitIgnoreSpec`) + by comparing their :attr:`~pathspec.pattern.Pattern` + attributes. A non-:class:`GitIgnoreSpec` will not compare equal. + """ + if isinstance(other, GitIgnoreSpec): + return super().__eq__(other) + elif isinstance(other, PathSpec): + return False + else: + return NotImplemented + + # Support reversed order of arguments from PathSpec. + @overload + @classmethod + def from_lines( + cls: Type[Self], + pattern_factory: Union[str, Callable[[AnyStr], Pattern]], + lines: Iterable[AnyStr], + ) -> Self: + ... + + @overload + @classmethod + def from_lines( + cls: Type[Self], + lines: Iterable[AnyStr], + pattern_factory: Union[str, Callable[[AnyStr], Pattern], None] = None, + ) -> Self: + ... + + @classmethod + def from_lines( + cls: Type[Self], + lines: Iterable[AnyStr], + pattern_factory: Union[str, Callable[[AnyStr], Pattern], None] = None, + ) -> Self: + """ + Compiles the pattern lines. + + *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled + pattern (:class:`str`). This simply has to yield each line so it can + be a :class:`io.TextIOBase` (e.g., from :func:`open` or + :class:`io.StringIO`) or the result from :meth:`str.splitlines`. + + *pattern_factory* can be :data:`None`, the name of a registered + pattern factory (:class:`str`), or a :class:`~collections.abc.Callable` + used to compile patterns. The callable must accept an uncompiled + pattern (:class:`str`) and return the compiled pattern + (:class:`pathspec.pattern.Pattern`). + Default is :data:`None` for :class:`.GitWildMatchPattern`). + + Returns the :class:`GitIgnoreSpec` instance. + """ + if pattern_factory is None: + pattern_factory = GitWildMatchPattern + + elif (isinstance(lines, (str, bytes)) or callable(lines)) and _is_iterable(pattern_factory): + # Support reversed order of arguments from PathSpec. + pattern_factory, lines = lines, pattern_factory + + self = super().from_lines(pattern_factory, lines) + return cast(Self, self) + + @staticmethod + def _match_file( + patterns: Iterable[Tuple[int, GitWildMatchPattern]], + file: str, + ) -> Tuple[Optional[bool], Optional[int]]: + """ + Check the file against the patterns. + + .. NOTE:: Subclasses of :class:`~pathspec.pathspec.PathSpec` may override + this method as an instance method. It does not have to be a static + method. The signature for this method is subject to change. + + *patterns* (:class:`~collections.abc.Iterable`) yields each indexed pattern + (:class:`tuple`) which contains the pattern index (:class:`int`) and actual + pattern (:class:`~pathspec.pattern.Pattern`). + + *file* (:class:`str`) is the normalized file path to be matched against + *patterns*. + + Returns a :class:`tuple` containing whether to include *file* (:class:`bool` + or :data:`None`), and the index of the last matched pattern (:class:`int` or + :data:`None`). + """ + out_include: Optional[bool] = None + out_index: Optional[int] = None + out_priority = 0 + for index, pattern in patterns: + if pattern.include is not None: + match = pattern.match_file(file) + if match is not None: + # Pattern matched. + + # Check for directory marker. + dir_mark = match.match.groupdict().get(_DIR_MARK) + + if dir_mark: + # Pattern matched by a directory pattern. + priority = 1 + else: + # Pattern matched by a file pattern. + priority = 2 + + if pattern.include and dir_mark: + out_include = pattern.include + out_index = index + out_priority = priority + elif priority >= out_priority: + out_include = pattern.include + out_index = index + out_priority = priority + + return out_include, out_index diff --git a/contrib/python/pathspec/pathspec/pathspec.py b/contrib/python/pathspec/pathspec/pathspec.py new file mode 100644 index 00000000000..bdfaccda495 --- /dev/null +++ b/contrib/python/pathspec/pathspec/pathspec.py @@ -0,0 +1,394 @@ +""" +This module provides an object oriented interface for pattern matching of files. +""" + +from collections.abc import ( + Collection as CollectionType) +from itertools import ( + zip_longest) +from typing import ( + AnyStr, + Callable, # Replaced by `collections.abc.Callable` in 3.9. + Collection, # Replaced by `collections.abc.Collection` in 3.9. + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Iterator, # Replaced by `collections.abc.Iterator` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Type, # Replaced by `type` in 3.9. + TypeVar, + Union) # Replaced by `X | Y` in 3.10. + +from . import util +from .pattern import ( + Pattern) +from .util import ( + CheckResult, + StrPath, + TStrPath, + TreeEntry, + _filter_check_patterns, + _is_iterable, + normalize_file) + +Self = TypeVar("Self", bound="PathSpec") +""" +:class:`PathSpec` self type hint to support Python v<3.11 using PEP 673 +recommendation. +""" + + +class PathSpec(object): + """ + The :class:`PathSpec` class is a wrapper around a list of compiled + :class:`.Pattern` instances. + """ + + def __init__(self, patterns: Iterable[Pattern]) -> None: + """ + Initializes the :class:`PathSpec` instance. + + *patterns* (:class:`~collections.abc.Collection` or :class:`~collections.abc.Iterable`) + yields each compiled pattern (:class:`.Pattern`). + """ + if not isinstance(patterns, CollectionType): + patterns = list(patterns) + + self.patterns: Collection[Pattern] = patterns + """ + *patterns* (:class:`~collections.abc.Collection` of :class:`.Pattern`) + contains the compiled patterns. + """ + + def __eq__(self, other: object) -> bool: + """ + Tests the equality of this path-spec with *other* (:class:`PathSpec`) + by comparing their :attr:`~PathSpec.patterns` attributes. + """ + if isinstance(other, PathSpec): + paired_patterns = zip_longest(self.patterns, other.patterns) + return all(a == b for a, b in paired_patterns) + else: + return NotImplemented + + def __len__(self) -> int: + """ + Returns the number of compiled patterns this path-spec contains + (:class:`int`). + """ + return len(self.patterns) + + def __add__(self: Self, other: "PathSpec") -> Self: + """ + Combines the :attr:`Pathspec.patterns` patterns from two + :class:`PathSpec` instances. + """ + if isinstance(other, PathSpec): + return self.__class__(self.patterns + other.patterns) + else: + return NotImplemented + + def __iadd__(self: Self, other: "PathSpec") -> Self: + """ + Adds the :attr:`Pathspec.patterns` patterns from one :class:`PathSpec` + instance to this instance. + """ + if isinstance(other, PathSpec): + self.patterns += other.patterns + return self + else: + return NotImplemented + + def check_file( + self, + file: TStrPath, + separators: Optional[Collection[str]] = None, + ) -> CheckResult[TStrPath]: + """ + Check the files against this path-spec. + + *file* (:class:`str` or :class:`os.PathLike`) is the file path to be + matched against :attr:`self.patterns <PathSpec.patterns>`. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + Returns the file check result (:class:`~pathspec.util.CheckResult`). + """ + norm_file = normalize_file(file, separators) + include, index = self._match_file(enumerate(self.patterns), norm_file) + return CheckResult(file, include, index) + + def check_files( + self, + files: Iterable[TStrPath], + separators: Optional[Collection[str]] = None, + ) -> Iterator[CheckResult[TStrPath]]: + """ + Check the files against this path-spec. + + *files* (:class:`~collections.abc.Iterable` of :class:`str` or + :class:`os.PathLike`) contains the file paths to be checked against + :attr:`self.patterns <PathSpec.patterns>`. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + Returns an :class:`~collections.abc.Iterator` yielding each file check + result (:class:`~pathspec.util.CheckResult`). + """ + if not _is_iterable(files): + raise TypeError(f"files:{files!r} is not an iterable.") + + use_patterns = _filter_check_patterns(self.patterns) + for orig_file in files: + norm_file = normalize_file(orig_file, separators) + include, index = self._match_file(use_patterns, norm_file) + yield CheckResult(orig_file, include, index) + + def check_tree_files( + self, + root: StrPath, + on_error: Optional[Callable[[OSError], None]] = None, + follow_links: Optional[bool] = None, + ) -> Iterator[CheckResult[str]]: + """ + Walks the specified root path for all files and checks them against this + path-spec. + + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search for files. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally + is the error handler for file-system exceptions. It will be called with the + exception (:exc:`OSError`). Reraise the exception to abort the walk. Default + is :data:`None` to ignore file-system exceptions. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk + symbolic links that resolve to directories. Default is :data:`None` for + :data:`True`. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + + Returns an :class:`~collections.abc.Iterator` yielding each file check + result (:class:`~pathspec.util.CheckResult`). + """ + files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) + yield from self.check_files(files) + + @classmethod + def from_lines( + cls: Type[Self], + pattern_factory: Union[str, Callable[[AnyStr], Pattern]], + lines: Iterable[AnyStr], + ) -> Self: + """ + Compiles the pattern lines. + + *pattern_factory* can be either the name of a registered pattern factory + (:class:`str`), or a :class:`~collections.abc.Callable` used to compile + patterns. It must accept an uncompiled pattern (:class:`str`) and return the + compiled pattern (:class:`.Pattern`). + + *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled pattern + (:class:`str`). This simply has to yield each line so that it can be a + :class:`io.TextIOBase` (e.g., from :func:`open` or :class:`io.StringIO`) or + the result from :meth:`str.splitlines`. + + Returns the :class:`PathSpec` instance. + """ + if isinstance(pattern_factory, str): + pattern_factory = util.lookup_pattern(pattern_factory) + + if not callable(pattern_factory): + raise TypeError(f"pattern_factory:{pattern_factory!r} is not callable.") + + if not _is_iterable(lines): + raise TypeError(f"lines:{lines!r} is not an iterable.") + + patterns = [pattern_factory(line) for line in lines if line] + return cls(patterns) + + def match_entries( + self, + entries: Iterable[TreeEntry], + separators: Optional[Collection[str]] = None, + *, + negate: Optional[bool] = None, + ) -> Iterator[TreeEntry]: + """ + Matches the entries to this path-spec. + + *entries* (:class:`~collections.abc.Iterable` of :class:`~pathspec.util.TreeEntry`) + contains the entries to be matched against :attr:`self.patterns <PathSpec.patterns>`. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + + Returns the matched entries (:class:`~collections.abc.Iterator` of + :class:`~pathspec.util.TreeEntry`). + """ + if not _is_iterable(entries): + raise TypeError(f"entries:{entries!r} is not an iterable.") + + use_patterns = _filter_check_patterns(self.patterns) + for entry in entries: + norm_file = normalize_file(entry.path, separators) + include, _index = self._match_file(use_patterns, norm_file) + + if negate: + include = not include + + if include: + yield entry + + _match_file = staticmethod(util.check_match_file) + """ + Match files using the `check_match_file()` utility function. Subclasses may + override this method as an instance method. It does not have to be a static + method. The signature for this method is subject to change. + """ + + def match_file( + self, + file: StrPath, + separators: Optional[Collection[str]] = None, + ) -> bool: + """ + Matches the file to this path-spec. + + *file* (:class:`str` or :class:`os.PathLike`) is the file path to be + matched against :attr:`self.patterns <PathSpec.patterns>`. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`) + optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + Returns :data:`True` if *file* matched; otherwise, :data:`False`. + """ + norm_file = normalize_file(file, separators) + include, _index = self._match_file(enumerate(self.patterns), norm_file) + return bool(include) + + def match_files( + self, + files: Iterable[StrPath], + separators: Optional[Collection[str]] = None, + *, + negate: Optional[bool] = None, + ) -> Iterator[StrPath]: + """ + Matches the files to this path-spec. + + *files* (:class:`~collections.abc.Iterable` of :class:`str` or + :class:`os.PathLike`) contains the file paths to be matched against + :attr:`self.patterns <PathSpec.patterns>`. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + + Returns the matched files (:class:`~collections.abc.Iterator` of + :class:`str` or :class:`os.PathLike`). + """ + if not _is_iterable(files): + raise TypeError(f"files:{files!r} is not an iterable.") + + use_patterns = _filter_check_patterns(self.patterns) + for orig_file in files: + norm_file = normalize_file(orig_file, separators) + include, _index = self._match_file(use_patterns, norm_file) + + if negate: + include = not include + + if include: + yield orig_file + + def match_tree_entries( + self, + root: StrPath, + on_error: Optional[Callable[[OSError], None]] = None, + follow_links: Optional[bool] = None, + *, + negate: Optional[bool] = None, + ) -> Iterator[TreeEntry]: + """ + Walks the specified root path for all files and matches them to this + path-spec. + + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally + is the error handler for file-system exceptions. It will be called with the + exception (:exc:`OSError`). Reraise the exception to abort the walk. Default + is :data:`None` to ignore file-system exceptions. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk + symbolic links that resolve to directories. Default is :data:`None` for + :data:`True`. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + + Returns the matched files (:class:`~collections.abc.Iterator` of + :class:`.TreeEntry`). + """ + entries = util.iter_tree_entries(root, on_error=on_error, follow_links=follow_links) + yield from self.match_entries(entries, negate=negate) + + def match_tree_files( + self, + root: StrPath, + on_error: Optional[Callable[[OSError], None]] = None, + follow_links: Optional[bool] = None, + *, + negate: Optional[bool] = None, + ) -> Iterator[str]: + """ + Walks the specified root path for all files and matches them to this + path-spec. + + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search for files. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally + is the error handler for file-system exceptions. It will be called with the + exception (:exc:`OSError`). Reraise the exception to abort the walk. Default + is :data:`None` to ignore file-system exceptions. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk + symbolic links that resolve to directories. Default is :data:`None` for + :data:`True`. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + + Returns the matched files (:class:`~collections.abc.Iterable` of + :class:`str`). + """ + files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) + yield from self.match_files(files, negate=negate) + + # Alias `match_tree_files()` as `match_tree()` for backward compatibility + # before v0.3.2. + match_tree = match_tree_files diff --git a/contrib/python/pathspec/pathspec/pattern.py b/contrib/python/pathspec/pathspec/pattern.py new file mode 100644 index 00000000000..d0815574837 --- /dev/null +++ b/contrib/python/pathspec/pathspec/pattern.py @@ -0,0 +1,213 @@ +""" +This module provides the base definition for patterns. +""" + +import dataclasses +import re +import warnings +from typing import ( + Any, + AnyStr, + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Iterator, # Replaced by `collections.abc.Iterator` in 3.9. + Match as MatchHint, # Replaced by `re.Match` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Pattern as PatternHint, # Replaced by `re.Pattern` in 3.9. + Tuple, # Replaced by `tuple` in 3.9. + Union) # Replaced by `X | Y` in 3.10. + + +class Pattern(object): + """ + The :class:`Pattern` class is the abstract definition of a pattern. + """ + + # Make the class dict-less. + __slots__ = ( + 'include', + ) + + def __init__(self, include: Optional[bool]) -> None: + """ + Initializes the :class:`Pattern` instance. + + *include* (:class:`bool` or :data:`None`) is whether the matched files + should be included (:data:`True`), excluded (:data:`False`), or is a + null-operation (:data:`None`). + """ + + self.include = include + """ + *include* (:class:`bool` or :data:`None`) is whether the matched files + should be included (:data:`True`), excluded (:data:`False`), or is a + null-operation (:data:`None`). + """ + + def match(self, files: Iterable[str]) -> Iterator[str]: + """ + DEPRECATED: This method is no longer used and has been replaced by + :meth:`.match_file`. Use the :meth:`.match_file` method with a loop for + similar results. + + Matches this pattern against the specified files. + + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains each + file relative to the root directory (e.g., ``"relative/path/to/file"``). + + Returns an :class:`~collections.abc.Iterable` yielding each matched file + path (:class:`str`). + """ + warnings.warn(( + "{cls.__module__}.{cls.__qualname__}.match() is deprecated. Use " + "{cls.__module__}.{cls.__qualname__}.match_file() with a loop for " + "similar results." + ).format(cls=self.__class__), DeprecationWarning, stacklevel=2) + + for file in files: + if self.match_file(file) is not None: + yield file + + def match_file(self, file: str) -> Optional[Any]: + """ + Matches this pattern against the specified file. + + *file* (:class:`str`) is the normalized file path to match against. + + Returns the match result if *file* matched; otherwise, :data:`None`. + """ + raise NotImplementedError(( + "{cls.__module__}.{cls.__qualname__} must override match_file()." + ).format(cls=self.__class__)) + + +class RegexPattern(Pattern): + """ + The :class:`RegexPattern` class is an implementation of a pattern using + regular expressions. + """ + + # Keep the class dict-less. + __slots__ = ( + 'pattern', + 'regex', + ) + + def __init__( + self, + pattern: Union[AnyStr, PatternHint, None], + include: Optional[bool] = None, + ) -> None: + """ + Initializes the :class:`RegexPattern` instance. + + *pattern* (:class:`str`, :class:`bytes`, :class:`re.Pattern`, or + :data:`None`) is the pattern to compile into a regular expression. + + *include* (:class:`bool` or :data:`None`) must be :data:`None` unless + *pattern* is a precompiled regular expression (:class:`re.Pattern`) in which + case it is whether matched files should be included (:data:`True`), excluded + (:data:`False`), or is a null operation (:data:`None`). + + .. NOTE:: Subclasses do not need to support the *include* parameter. + """ + + if isinstance(pattern, (str, bytes)): + assert include is None, ( + f"include:{include!r} must be null when pattern:{pattern!r} is a string." + ) + regex, include = self.pattern_to_regex(pattern) + # NOTE: Make sure to allow a null regular expression to be + # returned for a null-operation. + if include is not None: + regex = re.compile(regex) + + elif pattern is not None and hasattr(pattern, 'match'): + # Assume pattern is a precompiled regular expression. + # - NOTE: Used specified *include*. + regex = pattern + + elif pattern is None: + # NOTE: Make sure to allow a null pattern to be passed for a + # null-operation. + assert include is None, ( + f"include:{include!r} must be null when pattern:{pattern!r} is null." + ) + + else: + raise TypeError(f"pattern:{pattern!r} is not a string, re.Pattern, or None.") + + super(RegexPattern, self).__init__(include) + + self.pattern: Union[AnyStr, PatternHint, None] = pattern + """ + *pattern* (:class:`str`, :class:`bytes`, :class:`re.Pattern`, or + :data:`None`) is the uncompiled, input pattern. This is for reference. + """ + + self.regex: PatternHint = regex + """ + *regex* (:class:`re.Pattern`) is the regular expression for the pattern. + """ + + def __eq__(self, other: 'RegexPattern') -> bool: + """ + Tests the equality of this regex pattern with *other* (:class:`RegexPattern`) + by comparing their :attr:`~Pattern.include` and :attr:`~RegexPattern.regex` + attributes. + """ + if isinstance(other, RegexPattern): + return self.include == other.include and self.regex == other.regex + else: + return NotImplemented + + def match_file(self, file: str) -> Optional['RegexMatchResult']: + """ + Matches this pattern against the specified file. + + *file* (:class:`str`) contains each file relative to the root directory + (e.g., "relative/path/to/file"). + + Returns the match result (:class:`.RegexMatchResult`) if *file* matched; + otherwise, :data:`None`. + """ + if self.include is not None: + match = self.regex.match(file) + if match is not None: + return RegexMatchResult(match) + + return None + + @classmethod + def pattern_to_regex(cls, pattern: str) -> Tuple[str, bool]: + """ + Convert the pattern into an uncompiled regular expression. + + *pattern* (:class:`str`) is the pattern to convert into a regular + expression. + + Returns the uncompiled regular expression (:class:`str` or :data:`None`), + and whether matched files should be included (:data:`True`), excluded + (:data:`False`), or is a null-operation (:data:`None`). + + .. NOTE:: The default implementation simply returns *pattern* and + :data:`True`. + """ + return pattern, True + + +class RegexMatchResult(object): + """ + The :class:`RegexMatchResult` data class is used to return information about + the matched regular expression. + """ + + # Keep the class dict-less. + __slots__ = ( + 'match', + ) + + match: MatchHint + """ + *match* (:class:`re.Match`) is the regex match result. + """ diff --git a/contrib/python/pathspec/pathspec/patterns/__init__.py b/contrib/python/pathspec/pathspec/patterns/__init__.py new file mode 100644 index 00000000000..7360e9c2985 --- /dev/null +++ b/contrib/python/pathspec/pathspec/patterns/__init__.py @@ -0,0 +1,11 @@ +""" +The *pathspec.patterns* package contains the pattern matching +implementations. +""" + +# Load pattern implementations. +from . import gitwildmatch + +# DEPRECATED: Expose the `GitWildMatchPattern` class in this module for +# backward compatibility with v0.5. +from .gitwildmatch import GitWildMatchPattern diff --git a/contrib/python/pathspec/pathspec/patterns/gitwildmatch.py b/contrib/python/pathspec/pathspec/patterns/gitwildmatch.py new file mode 100644 index 00000000000..6a3d6d5eb05 --- /dev/null +++ b/contrib/python/pathspec/pathspec/patterns/gitwildmatch.py @@ -0,0 +1,421 @@ +""" +This module implements Git's wildmatch pattern matching which itself is derived +from Rsync's wildmatch. Git uses wildmatch for its ".gitignore" files. +""" + +import re +import warnings +from typing import ( + AnyStr, + Optional, # Replaced by `X | None` in 3.10. + Tuple) # Replaced by `tuple` in 3.9. + +from .. import util +from ..pattern import RegexPattern + +_BYTES_ENCODING = 'latin1' +""" +The encoding to use when parsing a byte string pattern. +""" + +_DIR_MARK = 'ps_d' +""" +The regex group name for the directory marker. This is only used by +:class:`GitIgnoreSpec`. +""" + + +class GitWildMatchPatternError(ValueError): + """ + The :class:`GitWildMatchPatternError` indicates an invalid git wild match + pattern. + """ + pass + + +class GitWildMatchPattern(RegexPattern): + """ + The :class:`GitWildMatchPattern` class represents a compiled Git wildmatch + pattern. + """ + + # Keep the dict-less class hierarchy. + __slots__ = () + + @classmethod + def pattern_to_regex( + cls, + pattern: AnyStr, + ) -> Tuple[Optional[AnyStr], Optional[bool]]: + """ + Convert the pattern into a regular expression. + + *pattern* (:class:`str` or :class:`bytes`) is the pattern to convert into a + regular expression. + + Returns the uncompiled regular expression (:class:`str`, :class:`bytes`, or + :data:`None`); and whether matched files should be included (:data:`True`), + excluded (:data:`False`), or if it is a null-operation (:data:`None`). + """ + if isinstance(pattern, str): + return_type = str + elif isinstance(pattern, bytes): + return_type = bytes + pattern = pattern.decode(_BYTES_ENCODING) + else: + raise TypeError(f"pattern:{pattern!r} is not a unicode or byte string.") + + original_pattern = pattern + + if pattern.endswith('\\ '): + # EDGE CASE: Spaces can be escaped with backslash. If a pattern that ends + # with backslash followed by a space, only strip from left. + pattern = pattern.lstrip() + else: + pattern = pattern.strip() + + if pattern.startswith('#'): + # A pattern starting with a hash ('#') serves as a comment (neither + # includes nor excludes files). Escape the hash with a back-slash to match + # a literal hash (i.e., '\#'). + regex = None + include = None + + elif pattern == '/': + # EDGE CASE: According to `git check-ignore` (v2.4.1), a single '/' does + # not match any file. + regex = None + include = None + + elif pattern: + if pattern.startswith('!'): + # A pattern starting with an exclamation mark ('!') negates the pattern + # (exclude instead of include). Escape the exclamation mark with a + # back-slash to match a literal exclamation mark (i.e., '\!'). + include = False + # Remove leading exclamation mark. + pattern = pattern[1:] + else: + include = True + + # Allow a regex override for edge cases that cannot be handled through + # normalization. + override_regex = None + + # Split pattern into segments. + pattern_segs = pattern.split('/') + + # Check whether the pattern is specifically a directory pattern before + # normalization. + is_dir_pattern = not pattern_segs[-1] + + # Normalize pattern to make processing easier. + + # EDGE CASE: Deal with duplicate double-asterisk sequences. Collapse each + # sequence down to one double-asterisk. Iterate over the segments in + # reverse and remove the duplicate double asterisks as we go. + for i in range(len(pattern_segs) - 1, 0, -1): + prev = pattern_segs[i-1] + seg = pattern_segs[i] + if prev == '**' and seg == '**': + del pattern_segs[i] + + if len(pattern_segs) == 2 and pattern_segs[0] == '**' and not pattern_segs[1]: + # EDGE CASE: The '**/' pattern should match everything except individual + # files in the root directory. This case cannot be adequately handled + # through normalization. Use the override. + override_regex = f'^.+(?P<{_DIR_MARK}>/).*$' + + if not pattern_segs[0]: + # A pattern beginning with a slash ('/') will only match paths directly + # on the root directory instead of any descendant paths. So, remove + # empty first segment to make pattern relative to root. + del pattern_segs[0] + + elif len(pattern_segs) == 1 or (len(pattern_segs) == 2 and not pattern_segs[1]): + # A single pattern without a beginning slash ('/') will match any + # descendant path. This is equivalent to "**/{pattern}". So, prepend + # with double-asterisks to make pattern relative to root. + # - EDGE CASE: This also holds for a single pattern with a trailing + # slash (e.g. dir/). + if pattern_segs[0] != '**': + pattern_segs.insert(0, '**') + + else: + # EDGE CASE: A pattern without a beginning slash ('/') but contains at + # least one prepended directory (e.g. "dir/{pattern}") should not match + # "**/dir/{pattern}", according to `git check-ignore` (v2.4.1). + pass + + if not pattern_segs: + # After resolving the edge cases, we end up with no pattern at all. This + # must be because the pattern is invalid. + raise GitWildMatchPatternError(f"Invalid git pattern: {original_pattern!r}") + + if not pattern_segs[-1] and len(pattern_segs) > 1: + # A pattern ending with a slash ('/') will match all descendant paths if + # it is a directory but not if it is a regular file. This is equivalent + # to "{pattern}/**". So, set last segment to a double-asterisk to + # include all descendants. + pattern_segs[-1] = '**' + + if override_regex is None: + # Build regular expression from pattern. + output = ['^'] + need_slash = False + end = len(pattern_segs) - 1 + for i, seg in enumerate(pattern_segs): + if seg == '**': + if i == 0 and i == end: + # A pattern consisting solely of double-asterisks ('**') will + # match every path. + output.append(f'[^/]+(?:/.*)?') + + elif i == 0: + # A normalized pattern beginning with double-asterisks + # ('**') will match any leading path segments. + output.append('(?:.+/)?') + need_slash = False + + elif i == end: + # A normalized pattern ending with double-asterisks ('**') will + # match any trailing path segments. + if is_dir_pattern: + output.append(f'(?P<{_DIR_MARK}>/).*') + else: + output.append(f'/.*') + + else: + # A pattern with inner double-asterisks ('**') will match multiple + # (or zero) inner path segments. + output.append('(?:/.+)?') + need_slash = True + + elif seg == '*': + # Match single path segment. + if need_slash: + output.append('/') + + output.append('[^/]+') + + if i == end: + # A pattern ending without a slash ('/') will match a file or a + # directory (with paths underneath it). E.g., "foo" matches "foo", + # "foo/bar", "foo/bar/baz", etc. + output.append(f'(?:(?P<{_DIR_MARK}>/).*)?') + + need_slash = True + + else: + # Match segment glob pattern. + if need_slash: + output.append('/') + + try: + output.append(cls._translate_segment_glob(seg)) + except ValueError as e: + raise GitWildMatchPatternError(f"Invalid git pattern: {original_pattern!r}") from e + + if i == end: + # A pattern ending without a slash ('/') will match a file or a + # directory (with paths underneath it). E.g., "foo" matches "foo", + # "foo/bar", "foo/bar/baz", etc. + output.append(f'(?:(?P<{_DIR_MARK}>/).*)?') + + need_slash = True + + output.append('$') + regex = ''.join(output) + + else: + # Use regex override. + regex = override_regex + + else: + # A blank pattern is a null-operation (neither includes nor excludes + # files). + regex = None + include = None + + if regex is not None and return_type is bytes: + regex = regex.encode(_BYTES_ENCODING) + + return regex, include + + @staticmethod + def _translate_segment_glob(pattern: str) -> str: + """ + Translates the glob pattern to a regular expression. This is used in the + constructor to translate a path segment glob pattern to its corresponding + regular expression. + + *pattern* (:class:`str`) is the glob pattern. + + Returns the regular expression (:class:`str`). + """ + # NOTE: This is derived from `fnmatch.translate()` and is similar to the + # POSIX function `fnmatch()` with the `FNM_PATHNAME` flag set. + + escape = False + regex = '' + i, end = 0, len(pattern) + while i < end: + # Get next character. + char = pattern[i] + i += 1 + + if escape: + # Escape the character. + escape = False + regex += re.escape(char) + + elif char == '\\': + # Escape character, escape next character. + escape = True + + elif char == '*': + # Multi-character wildcard. Match any string (except slashes), including + # an empty string. + regex += '[^/]*' + + elif char == '?': + # Single-character wildcard. Match any single character (except a + # slash). + regex += '[^/]' + + elif char == '[': + # Bracket expression wildcard. Except for the beginning exclamation + # mark, the whole bracket expression can be used directly as regex, but + # we have to find where the expression ends. + # - "[][!]" matches ']', '[' and '!'. + # - "[]-]" matches ']' and '-'. + # - "[!]a-]" matches any character except ']', 'a' and '-'. + j = i + + # Pass bracket expression negation. + if j < end and (pattern[j] == '!' or pattern[j] == '^'): + j += 1 + + # Pass first closing bracket if it is at the beginning of the + # expression. + if j < end and pattern[j] == ']': + j += 1 + + # Find closing bracket. Stop once we reach the end or find it. + while j < end and pattern[j] != ']': + j += 1 + + if j < end: + # Found end of bracket expression. Increment j to be one past the + # closing bracket: + # + # [...] + # ^ ^ + # i j + # + j += 1 + expr = '[' + + if pattern[i] == '!': + # Bracket expression needs to be negated. + expr += '^' + i += 1 + elif pattern[i] == '^': + # POSIX declares that the regex bracket expression negation "[^...]" + # is undefined in a glob pattern. Python's `fnmatch.translate()` + # escapes the caret ('^') as a literal. Git supports the using a + # caret for negation. Maintain consistency with Git because that is + # the expected behavior. + expr += '^' + i += 1 + + # Build regex bracket expression. Escape slashes so they are treated + # as literal slashes by regex as defined by POSIX. + expr += pattern[i:j].replace('\\', '\\\\') + + # Add regex bracket expression to regex result. + regex += expr + + # Set i to one past the closing bracket. + i = j + + else: + # Failed to find closing bracket, treat opening bracket as a bracket + # literal instead of as an expression. + regex += '\\[' + + else: + # Regular character, escape it for regex. + regex += re.escape(char) + + if escape: + raise ValueError(f"Escape character found with no next character to escape: {pattern!r}") + + return regex + + @staticmethod + def escape(s: AnyStr) -> AnyStr: + """ + Escape special characters in the given string. + + *s* (:class:`str` or :class:`bytes`) a filename or a string that you want to + escape, usually before adding it to a ".gitignore". + + Returns the escaped string (:class:`str` or :class:`bytes`). + """ + if isinstance(s, str): + return_type = str + string = s + elif isinstance(s, bytes): + return_type = bytes + string = s.decode(_BYTES_ENCODING) + else: + raise TypeError(f"s:{s!r} is not a unicode or byte string.") + + # Reference: https://git-scm.com/docs/gitignore#_pattern_format + meta_characters = r"[]!*#?" + + out_string = "".join("\\" + x if x in meta_characters else x for x in string) + + if return_type is bytes: + return out_string.encode(_BYTES_ENCODING) + else: + return out_string + +util.register_pattern('gitwildmatch', GitWildMatchPattern) + + +class GitIgnorePattern(GitWildMatchPattern): + """ + The :class:`GitIgnorePattern` class is deprecated by :class:`GitWildMatchPattern`. + This class only exists to maintain compatibility with v0.4. + """ + + def __init__(self, *args, **kw) -> None: + """ + Warn about deprecation. + """ + self._deprecated() + super(GitIgnorePattern, self).__init__(*args, **kw) + + @staticmethod + def _deprecated() -> None: + """ + Warn about deprecation. + """ + warnings.warn(( + "GitIgnorePattern ('gitignore') is deprecated. Use GitWildMatchPattern " + "('gitwildmatch') instead." + ), DeprecationWarning, stacklevel=3) + + @classmethod + def pattern_to_regex(cls, *args, **kw): + """ + Warn about deprecation. + """ + cls._deprecated() + return super(GitIgnorePattern, cls).pattern_to_regex(*args, **kw) + +# Register `GitIgnorePattern` as "gitignore" for backward compatibility with +# v0.4. +util.register_pattern('gitignore', GitIgnorePattern) diff --git a/contrib/python/pathspec/pathspec/py.typed b/contrib/python/pathspec/pathspec/py.typed new file mode 100644 index 00000000000..b01eaaf7abc --- /dev/null +++ b/contrib/python/pathspec/pathspec/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. The pathspec package uses inline types. diff --git a/contrib/python/pathspec/pathspec/util.py b/contrib/python/pathspec/pathspec/util.py new file mode 100644 index 00000000000..58839511b84 --- /dev/null +++ b/contrib/python/pathspec/pathspec/util.py @@ -0,0 +1,792 @@ +""" +This module provides utility methods for dealing with path-specs. +""" + +import os +import os.path +import pathlib +import posixpath +import stat +import sys +import warnings +from collections.abc import ( + Collection as CollectionType, + Iterable as IterableType) +from dataclasses import ( + dataclass) +from os import ( + PathLike) +from typing import ( + Any, + AnyStr, + Callable, # Replaced by `collections.abc.Callable` in 3.9. + Collection, # Replaced by `collections.abc.Collection` in 3.9. + Dict, # Replaced by `dict` in 3.9. + Generic, + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Iterator, # Replaced by `collections.abc.Iterator` in 3.9. + List, # Replaced by `list` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Sequence, # Replaced by `collections.abc.Sequence` in 3.9. + Set, # Replaced by `set` in 3.9. + Tuple, # Replaced by `tuple` in 3.9. + TypeVar, + Union) # Replaced by `X | Y` in 3.10. + +from .pattern import ( + Pattern) + +if sys.version_info >= (3, 9): + StrPath = Union[str, PathLike[str]] +else: + StrPath = Union[str, PathLike] + +TStrPath = TypeVar("TStrPath", bound=StrPath) +""" +Type variable for :class:`str` or :class:`os.PathLike`. +""" + +NORMALIZE_PATH_SEPS = [ + __sep + for __sep in [os.sep, os.altsep] + if __sep and __sep != posixpath.sep +] +""" +*NORMALIZE_PATH_SEPS* (:class:`list` of :class:`str`) contains the path +separators that need to be normalized to the POSIX separator for the +current operating system. The separators are determined by examining +:data:`os.sep` and :data:`os.altsep`. +""" + +_registered_patterns = {} +""" +*_registered_patterns* (:class:`dict`) maps a name (:class:`str`) to the +registered pattern factory (:class:`~collections.abc.Callable`). +""" + + +def append_dir_sep(path: pathlib.Path) -> str: + """ + Appends the path separator to the path if the path is a directory. + This can be used to aid in distinguishing between directories and + files on the file-system by relying on the presence of a trailing path + separator. + + *path* (:class:`pathlib.Path`) is the path to use. + + Returns the path (:class:`str`). + """ + str_path = str(path) + if path.is_dir(): + str_path += os.sep + + return str_path + + +def check_match_file( + patterns: Iterable[Tuple[int, Pattern]], + file: str, +) -> Tuple[Optional[bool], Optional[int]]: + """ + Check the file against the patterns. + + *patterns* (:class:`~collections.abc.Iterable`) yields each indexed pattern + (:class:`tuple`) which contains the pattern index (:class:`int`) and actual + pattern (:class:`~pathspec.pattern.Pattern`). + + *file* (:class:`str`) is the normalized file path to be matched + against *patterns*. + + Returns a :class:`tuple` containing whether to include *file* (:class:`bool` + or :data:`None`), and the index of the last matched pattern (:class:`int` or + :data:`None`). + """ + out_include: Optional[bool] = None + out_index: Optional[int] = None + for index, pattern in patterns: + if pattern.include is not None and pattern.match_file(file) is not None: + out_include = pattern.include + out_index = index + + return out_include, out_index + + +def detailed_match_files( + patterns: Iterable[Pattern], + files: Iterable[str], + all_matches: Optional[bool] = None, +) -> Dict[str, 'MatchDetail']: + """ + Matches the files to the patterns, and returns which patterns matched + the files. + + *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) + contains the patterns to use. + + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains + the normalized file paths to be matched against *patterns*. + + *all_matches* (:class:`bool` or :data:`None`) is whether to return all + matches patterns (:data:`True`), or only the last matched pattern + (:data:`False`). Default is :data:`None` for :data:`False`. + + Returns the matched files (:class:`dict`) which maps each matched file + (:class:`str`) to the patterns that matched in order (:class:`.MatchDetail`). + """ + all_files = files if isinstance(files, CollectionType) else list(files) + return_files = {} + for pattern in patterns: + if pattern.include is not None: + result_files = pattern.match(all_files) # TODO: Replace with `.match_file()`. + if pattern.include: + # Add files and record pattern. + for result_file in result_files: + if result_file in return_files: + if all_matches: + return_files[result_file].patterns.append(pattern) + else: + return_files[result_file].patterns[0] = pattern + else: + return_files[result_file] = MatchDetail([pattern]) + + else: + # Remove files. + for file in result_files: + del return_files[file] + + return return_files + + +def _filter_check_patterns( + patterns: Iterable[Pattern], +) -> List[Tuple[int, Pattern]]: + """ + Filters out null-patterns. + + *patterns* (:class:`Iterable` of :class:`.Pattern`) contains the + patterns. + + Returns a :class:`list` containing each indexed pattern (:class:`tuple`) which + contains the pattern index (:class:`int`) and the actual pattern + (:class:`~pathspec.pattern.Pattern`). + """ + return [ + (__index, __pat) + for __index, __pat in enumerate(patterns) + if __pat.include is not None + ] + + +def _is_iterable(value: Any) -> bool: + """ + Check whether the value is an iterable (excludes strings). + + *value* is the value to check, + + Returns whether *value* is a iterable (:class:`bool`). + """ + return isinstance(value, IterableType) and not isinstance(value, (str, bytes)) + + +def iter_tree_entries( + root: StrPath, + on_error: Optional[Callable[[OSError], None]] = None, + follow_links: Optional[bool] = None, +) -> Iterator['TreeEntry']: + """ + Walks the specified directory for all files and directories. + + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. It will be + called with the exception (:exc:`OSError`). Reraise the exception to + abort the walk. Default is :data:`None` to ignore file-system + exceptions. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether + to walk symbolic links that resolve to directories. Default is + :data:`None` for :data:`True`. + + Raises :exc:`RecursionError` if recursion is detected. + + Returns an :class:`~collections.abc.Iterator` yielding each file or + directory entry (:class:`.TreeEntry`) relative to *root*. + """ + if on_error is not None and not callable(on_error): + raise TypeError(f"on_error:{on_error!r} is not callable.") + + if follow_links is None: + follow_links = True + + yield from _iter_tree_entries_next(os.path.abspath(root), '', {}, on_error, follow_links) + + +def _iter_tree_entries_next( + root_full: str, + dir_rel: str, + memo: Dict[str, str], + on_error: Callable[[OSError], None], + follow_links: bool, +) -> Iterator['TreeEntry']: + """ + Scan the directory for all descendant files. + + *root_full* (:class:`str`) the absolute path to the root directory. + + *dir_rel* (:class:`str`) the path to the directory to scan relative to + *root_full*. + + *memo* (:class:`dict`) keeps track of ancestor directories + encountered. Maps each ancestor real path (:class:`str`) to relative + path (:class:`str`). + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. + + *follow_links* (:class:`bool`) is whether to walk symbolic links that + resolve to directories. + + Yields each entry (:class:`.TreeEntry`). + """ + dir_full = os.path.join(root_full, dir_rel) + dir_real = os.path.realpath(dir_full) + + # Remember each encountered ancestor directory and its canonical + # (real) path. If a canonical path is encountered more than once, + # recursion has occurred. + if dir_real not in memo: + memo[dir_real] = dir_rel + else: + raise RecursionError(real_path=dir_real, first_path=memo[dir_real], second_path=dir_rel) + + with os.scandir(dir_full) as scan_iter: + node_ent: os.DirEntry + for node_ent in scan_iter: + node_rel = os.path.join(dir_rel, node_ent.name) + + # Inspect child node. + try: + node_lstat = node_ent.stat(follow_symlinks=False) + except OSError as e: + if on_error is not None: + on_error(e) + continue + + if node_ent.is_symlink(): + # Child node is a link, inspect the target node. + try: + node_stat = node_ent.stat() + except OSError as e: + if on_error is not None: + on_error(e) + continue + else: + node_stat = node_lstat + + if node_ent.is_dir(follow_symlinks=follow_links): + # Child node is a directory, recurse into it and yield its + # descendant files. + yield TreeEntry(node_ent.name, node_rel, node_lstat, node_stat) + + yield from _iter_tree_entries_next(root_full, node_rel, memo, on_error, follow_links) + + elif node_ent.is_file() or node_ent.is_symlink(): + # Child node is either a file or an unfollowed link, yield it. + yield TreeEntry(node_ent.name, node_rel, node_lstat, node_stat) + + # NOTE: Make sure to remove the canonical (real) path of the directory + # from the ancestors memo once we are done with it. This allows the + # same directory to appear multiple times. If this is not done, the + # second occurrence of the directory will be incorrectly interpreted + # as a recursion. See <https://github.com/cpburnz/python-path-specification/pull/7>. + del memo[dir_real] + + +def iter_tree_files( + root: StrPath, + on_error: Optional[Callable[[OSError], None]] = None, + follow_links: Optional[bool] = None, +) -> Iterator[str]: + """ + Walks the specified directory for all files. + + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search for files. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. It will be + called with the exception (:exc:`OSError`). Reraise the exception to + abort the walk. Default is :data:`None` to ignore file-system + exceptions. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether + to walk symbolic links that resolve to directories. Default is + :data:`None` for :data:`True`. + + Raises :exc:`RecursionError` if recursion is detected. + + Returns an :class:`~collections.abc.Iterator` yielding the path to + each file (:class:`str`) relative to *root*. + """ + for entry in iter_tree_entries(root, on_error=on_error, follow_links=follow_links): + if not entry.is_dir(follow_links): + yield entry.path + + +def iter_tree(root, on_error=None, follow_links=None): + """ + DEPRECATED: The :func:`.iter_tree` function is an alias for the + :func:`.iter_tree_files` function. + """ + warnings.warn(( + "util.iter_tree() is deprecated. Use util.iter_tree_files() instead." + ), DeprecationWarning, stacklevel=2) + return iter_tree_files(root, on_error=on_error, follow_links=follow_links) + + +def lookup_pattern(name: str) -> Callable[[AnyStr], Pattern]: + """ + Lookups a registered pattern factory by name. + + *name* (:class:`str`) is the name of the pattern factory. + + Returns the registered pattern factory (:class:`~collections.abc.Callable`). + If no pattern factory is registered, raises :exc:`KeyError`. + """ + return _registered_patterns[name] + + +def match_file(patterns: Iterable[Pattern], file: str) -> bool: + """ + Matches the file to the patterns. + + *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) + contains the patterns to use. + + *file* (:class:`str`) is the normalized file path to be matched + against *patterns*. + + Returns :data:`True` if *file* matched; otherwise, :data:`False`. + """ + matched = False + for pattern in patterns: + if pattern.include is not None and pattern.match_file(file) is not None: + matched = pattern.include + + return matched + + +def match_files( + patterns: Iterable[Pattern], + files: Iterable[str], +) -> Set[str]: + """ + DEPRECATED: This is an old function no longer used. Use the + :func:`~pathspec.util.match_file` function with a loop for better results. + + Matches the files to the patterns. + + *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) + contains the patterns to use. + + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains + the normalized file paths to be matched against *patterns*. + + Returns the matched files (:class:`set` of :class:`str`). + """ + warnings.warn(( + f"{__name__}.match_files() is deprecated. Use {__name__}.match_file() with " + f"a loop for better results." + ), DeprecationWarning, stacklevel=2) + + use_patterns = [__pat for __pat in patterns if __pat.include is not None] + + return_files = set() + for file in files: + if match_file(use_patterns, file): + return_files.add(file) + + return return_files + + +def normalize_file( + file: StrPath, + separators: Optional[Collection[str]] = None, +) -> str: + """ + Normalizes the file path to use the POSIX path separator (i.e., + ``"/"``), and make the paths relative (remove leading ``"/"``). + + *file* (:class:`str` or :class:`os.PathLike`) is the file path. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + ``None``) optionally contains the path separators to normalize. + This does not need to include the POSIX path separator (``"/"``), + but including it will not affect the results. Default is ``None`` + for ``NORMALIZE_PATH_SEPS``. To prevent normalization, pass an + empty container (e.g., an empty tuple ``()``). + + Returns the normalized file path (:class:`str`). + """ + # Normalize path separators. + if separators is None: + separators = NORMALIZE_PATH_SEPS + + # Convert path object to string. + norm_file: str = os.fspath(file) + + for sep in separators: + norm_file = norm_file.replace(sep, posixpath.sep) + + if norm_file.startswith('/'): + # Make path relative. + norm_file = norm_file[1:] + + elif norm_file.startswith('./'): + # Remove current directory prefix. + norm_file = norm_file[2:] + + return norm_file + + +def normalize_files( + files: Iterable[StrPath], + separators: Optional[Collection[str]] = None, +) -> Dict[str, List[StrPath]]: + """ + DEPRECATED: This function is no longer used. Use the :func:`.normalize_file` + function with a loop for better results. + + Normalizes the file paths to use the POSIX path separator. + + *files* (:class:`~collections.abc.Iterable` of :class:`str` or + :class:`os.PathLike`) contains the file paths to be normalized. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. + See :func:`normalize_file` for more information. + + Returns a :class:`dict` mapping each normalized file path (:class:`str`) + to the original file paths (:class:`list` of :class:`str` or + :class:`os.PathLike`). + """ + warnings.warn(( + "util.normalize_files() is deprecated. Use util.normalize_file() " + "with a loop for better results." + ), DeprecationWarning, stacklevel=2) + + norm_files = {} + for path in files: + norm_file = normalize_file(path, separators=separators) + if norm_file in norm_files: + norm_files[norm_file].append(path) + else: + norm_files[norm_file] = [path] + + return norm_files + + +def register_pattern( + name: str, + pattern_factory: Callable[[AnyStr], Pattern], + override: Optional[bool] = None, +) -> None: + """ + Registers the specified pattern factory. + + *name* (:class:`str`) is the name to register the pattern factory + under. + + *pattern_factory* (:class:`~collections.abc.Callable`) is used to + compile patterns. It must accept an uncompiled pattern (:class:`str`) + and return the compiled pattern (:class:`.Pattern`). + + *override* (:class:`bool` or :data:`None`) optionally is whether to + allow overriding an already registered pattern under the same name + (:data:`True`), instead of raising an :exc:`AlreadyRegisteredError` + (:data:`False`). Default is :data:`None` for :data:`False`. + """ + if not isinstance(name, str): + raise TypeError(f"name:{name!r} is not a string.") + + if not callable(pattern_factory): + raise TypeError(f"pattern_factory:{pattern_factory!r} is not callable.") + + if name in _registered_patterns and not override: + raise AlreadyRegisteredError(name, _registered_patterns[name]) + + _registered_patterns[name] = pattern_factory + + +class AlreadyRegisteredError(Exception): + """ + The :exc:`AlreadyRegisteredError` exception is raised when a pattern + factory is registered under a name already in use. + """ + + def __init__( + self, + name: str, + pattern_factory: Callable[[AnyStr], Pattern], + ) -> None: + """ + Initializes the :exc:`AlreadyRegisteredError` instance. + + *name* (:class:`str`) is the name of the registered pattern. + + *pattern_factory* (:class:`~collections.abc.Callable`) is the + registered pattern factory. + """ + super(AlreadyRegisteredError, self).__init__(name, pattern_factory) + + @property + def message(self) -> str: + """ + *message* (:class:`str`) is the error message. + """ + return "{name!r} is already registered for pattern factory:{pattern_factory!r}.".format( + name=self.name, + pattern_factory=self.pattern_factory, + ) + + @property + def name(self) -> str: + """ + *name* (:class:`str`) is the name of the registered pattern. + """ + return self.args[0] + + @property + def pattern_factory(self) -> Callable[[AnyStr], Pattern]: + """ + *pattern_factory* (:class:`~collections.abc.Callable`) is the + registered pattern factory. + """ + return self.args[1] + + +class RecursionError(Exception): + """ + The :exc:`RecursionError` exception is raised when recursion is + detected. + """ + + def __init__( + self, + real_path: str, + first_path: str, + second_path: str, + ) -> None: + """ + Initializes the :exc:`RecursionError` instance. + + *real_path* (:class:`str`) is the real path that recursion was + encountered on. + + *first_path* (:class:`str`) is the first path encountered for + *real_path*. + + *second_path* (:class:`str`) is the second path encountered for + *real_path*. + """ + super(RecursionError, self).__init__(real_path, first_path, second_path) + + @property + def first_path(self) -> str: + """ + *first_path* (:class:`str`) is the first path encountered for + :attr:`self.real_path <RecursionError.real_path>`. + """ + return self.args[1] + + @property + def message(self) -> str: + """ + *message* (:class:`str`) is the error message. + """ + return "Real path {real!r} was encountered at {first!r} and then {second!r}.".format( + real=self.real_path, + first=self.first_path, + second=self.second_path, + ) + + @property + def real_path(self) -> str: + """ + *real_path* (:class:`str`) is the real path that recursion was + encountered on. + """ + return self.args[0] + + @property + def second_path(self) -> str: + """ + *second_path* (:class:`str`) is the second path encountered for + :attr:`self.real_path <RecursionError.real_path>`. + """ + return self.args[2] + + +@dataclass(frozen=True) +class CheckResult(Generic[TStrPath]): + """ + The :class:`CheckResult` class contains information about the file and which + pattern matched it. + """ + + # Make the class dict-less. + __slots__ = ( + 'file', + 'include', + 'index', + ) + + file: TStrPath + """ + *file* (:class:`str` or :class:`os.PathLike`) is the file path. + """ + + include: Optional[bool] + """ + *include* (:class:`bool` or :data:`None`) is whether to include or exclude the + file. If :data:`None`, no pattern matched. + """ + + index: Optional[int] + """ + *index* (:class:`int` or :data:`None`) is the index of the last pattern that + matched. If :data:`None`, no pattern matched. + """ + + +class MatchDetail(object): + """ + The :class:`.MatchDetail` class contains information about + """ + + # Make the class dict-less. + __slots__ = ('patterns',) + + def __init__(self, patterns: Sequence[Pattern]) -> None: + """ + Initialize the :class:`.MatchDetail` instance. + + *patterns* (:class:`~collections.abc.Sequence` of :class:`~pathspec.pattern.Pattern`) + contains the patterns that matched the file in the order they were + encountered. + """ + + self.patterns = patterns + """ + *patterns* (:class:`~collections.abc.Sequence` of :class:`~pathspec.pattern.Pattern`) + contains the patterns that matched the file in the order they were + encountered. + """ + + +class TreeEntry(object): + """ + The :class:`.TreeEntry` class contains information about a file-system + entry. + """ + + # Make the class dict-less. + __slots__ = ('_lstat', 'name', 'path', '_stat') + + def __init__( + self, + name: str, + path: str, + lstat: os.stat_result, + stat: os.stat_result, + ) -> None: + """ + Initialize the :class:`.TreeEntry` instance. + + *name* (:class:`str`) is the base name of the entry. + + *path* (:class:`str`) is the relative path of the entry. + + *lstat* (:class:`os.stat_result`) is the stat result of the direct + entry. + + *stat* (:class:`os.stat_result`) is the stat result of the entry, + potentially linked. + """ + + self._lstat: os.stat_result = lstat + """ + *_lstat* (:class:`os.stat_result`) is the stat result of the direct + entry. + """ + + self.name: str = name + """ + *name* (:class:`str`) is the base name of the entry. + """ + + self.path: str = path + """ + *path* (:class:`str`) is the path of the entry. + """ + + self._stat: os.stat_result = stat + """ + *_stat* (:class:`os.stat_result`) is the stat result of the linked + entry. + """ + + def is_dir(self, follow_links: Optional[bool] = None) -> bool: + """ + Get whether the entry is a directory. + + *follow_links* (:class:`bool` or :data:`None`) is whether to follow + symbolic links. If this is :data:`True`, a symlink to a directory + will result in :data:`True`. Default is :data:`None` for :data:`True`. + + Returns whether the entry is a directory (:class:`bool`). + """ + if follow_links is None: + follow_links = True + + node_stat = self._stat if follow_links else self._lstat + return stat.S_ISDIR(node_stat.st_mode) + + def is_file(self, follow_links: Optional[bool] = None) -> bool: + """ + Get whether the entry is a regular file. + + *follow_links* (:class:`bool` or :data:`None`) is whether to follow + symbolic links. If this is :data:`True`, a symlink to a regular file + will result in :data:`True`. Default is :data:`None` for :data:`True`. + + Returns whether the entry is a regular file (:class:`bool`). + """ + if follow_links is None: + follow_links = True + + node_stat = self._stat if follow_links else self._lstat + return stat.S_ISREG(node_stat.st_mode) + + def is_symlink(self) -> bool: + """ + Returns whether the entry is a symbolic link (:class:`bool`). + """ + return stat.S_ISLNK(self._lstat.st_mode) + + def stat(self, follow_links: Optional[bool] = None) -> os.stat_result: + """ + Get the cached stat result for the entry. + + *follow_links* (:class:`bool` or :data:`None`) is whether to follow + symbolic links. If this is :data:`True`, the stat result of the + linked file will be returned. Default is :data:`None` for :data:`True`. + + Returns that stat result (:class:`os.stat_result`). + """ + if follow_links is None: + follow_links = True + + return self._stat if follow_links else self._lstat diff --git a/contrib/python/pathspec/ya.make b/contrib/python/pathspec/ya.make new file mode 100644 index 00000000000..e5f440c174c --- /dev/null +++ b/contrib/python/pathspec/ya.make @@ -0,0 +1,30 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(0.12.1) + +LICENSE(MPL-2.0) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + pathspec/__init__.py + pathspec/_meta.py + pathspec/gitignore.py + pathspec/pathspec.py + pathspec/pattern.py + pathspec/patterns/__init__.py + pathspec/patterns/gitwildmatch.py + pathspec/util.py +) + +RESOURCE_FILES( + PREFIX contrib/python/pathspec/ + .dist-info/METADATA + .dist-info/top_level.txt + pathspec/py.typed +) + +END() diff --git a/contrib/python/portalocker/py2/.dist-info/METADATA b/contrib/python/portalocker/py2/.dist-info/METADATA new file mode 100644 index 00000000000..d01a6203e0a --- /dev/null +++ b/contrib/python/portalocker/py2/.dist-info/METADATA @@ -0,0 +1,136 @@ +Metadata-Version: 2.1 +Name: portalocker +Version: 1.7.1 +Summary: Wraps the portalocker recipe for easy usage +Home-page: https://github.com/WoLpH/portalocker +Author: Rick van Hattem +Author-email: [email protected] +License: PSF +Keywords: locking,locks,with statement,windows,linux,unix +Platform: any +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Dist: pywin32 (!=226) ; platform_system == "Windows" +Provides-Extra: docs +Requires-Dist: sphinx (>=1.7.1) ; extra == 'docs' +Provides-Extra: tests +Requires-Dist: pytest (>=4.6.9) ; extra == 'tests' +Requires-Dist: pytest-cov (>=2.8.1) ; extra == 'tests' +Requires-Dist: sphinx (>=1.8.5) ; extra == 'tests' +Requires-Dist: pytest-flake8 (>=1.0.5) ; extra == 'tests' + +############################################ +portalocker - Cross-platform locking library +############################################ + +.. image:: https://travis-ci.org/WoLpH/portalocker.svg?branch=master + :alt: Linux Test Status + :target: https://travis-ci.org/WoLpH/portalocker + +.. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true + :alt: Windows Tests Status + :target: https://ci.appveyor.com/project/WoLpH/portalocker + +.. image:: https://coveralls.io/repos/WoLpH/portalocker/badge.svg?branch=master + :alt: Coverage Status + :target: https://coveralls.io/r/WoLpH/portalocker?branch=master + +Overview +-------- + +Portalocker is a library to provide an easy API to file locking. + +An important detail to note is that on Linux and Unix systems the locks are +advisory by default. By specifying the `-o mand` option to the mount command it +is possible to enable mandatory file locking on Linux. This is generally not +recommended however. For more information about the subject: + + - https://en.wikipedia.org/wiki/File_locking + - http://stackoverflow.com/questions/39292051/portalocker-does-not-seem-to-lock + - https://stackoverflow.com/questions/12062466/mandatory-file-lock-on-linux + +The module is currently maintained by Rick van Hattem <[email protected]>. +The project resides at https://github.com/WoLpH/portalocker . Bugs and feature +requests can be submitted there. Patches are also very welcome. + +Tips +---- + +On some networked filesystems it might be needed to force a `os.fsync()` before +closing the file so it's actually written before another client reads the file. +Effectively this comes down to: + +:: + + with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: + # do what you need to do + ... + + # flush and sync to filesystem + fh.flush() + os.fsync(fh.fileno()) + +Links +----- + +* Documentation + - http://portalocker.readthedocs.org/en/latest/ +* Source + - https://github.com/WoLpH/portalocker +* Bug reports + - https://github.com/WoLpH/portalocker/issues +* Package homepage + - https://pypi.python.org/pypi/portalocker +* My blog + - http://w.wol.ph/ + +Examples +-------- + +To make sure your cache generation scripts don't race, use the `Lock` class: + +>>> import portalocker +>>> with portalocker.Lock('somefile', timeout=1) as fh: + print >>fh, 'writing some stuff to my cache...' + +To customize the opening and locking a manual approach is also possible: + +>>> import portalocker +>>> file = open('somefile', 'r+') +>>> portalocker.lock(file, portalocker.LOCK_EX) +>>> file.seek(12) +>>> file.write('foo') +>>> file.close() + +Explicitly unlocking might not be needed in all cases: +https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 + +But can be done through: + +>>> portalocker.unlock(file) + +Do note that your data might still be in a buffer so it is possible that your +data is not available until you `flush()` or `close()`. + +More examples can be found in the +`tests <http://portalocker.readthedocs.io/en/latest/_modules/tests/tests.html>`_. + +Changelog +--------- + +See the `changelog <http://portalocker.readthedocs.io/en/latest/changelog.html>`_ page. + +License +------- + +See the `LICENSE <https://github.com/WoLpH/portalocker/blob/develop/LICENSE>`_ file. + + + diff --git a/contrib/python/portalocker/py2/.dist-info/top_level.txt b/contrib/python/portalocker/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..7bbc14e6fa6 --- /dev/null +++ b/contrib/python/portalocker/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +portalocker diff --git a/contrib/python/portalocker/py2/.yandex_meta/yamaker.yaml b/contrib/python/portalocker/py2/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..bca0c4e5337 --- /dev/null +++ b/contrib/python/portalocker/py2/.yandex_meta/yamaker.yaml @@ -0,0 +1,4 @@ +keep: + - tests/* +exclude: + - portalocker_tests/* diff --git a/contrib/python/portalocker/py2/LICENSE b/contrib/python/portalocker/py2/LICENSE new file mode 100644 index 00000000000..adb8038169c --- /dev/null +++ b/contrib/python/portalocker/py2/LICENSE @@ -0,0 +1,48 @@ +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 +Python Software Foundation; All Rights Reserved" are retained in Python alone or +in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + diff --git a/contrib/python/portalocker/py2/README.rst b/contrib/python/portalocker/py2/README.rst new file mode 100644 index 00000000000..c013490c765 --- /dev/null +++ b/contrib/python/portalocker/py2/README.rst @@ -0,0 +1,106 @@ +############################################ +portalocker - Cross-platform locking library +############################################ + +.. image:: https://travis-ci.org/WoLpH/portalocker.svg?branch=master + :alt: Linux Test Status + :target: https://travis-ci.org/WoLpH/portalocker + +.. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true + :alt: Windows Tests Status + :target: https://ci.appveyor.com/project/WoLpH/portalocker + +.. image:: https://coveralls.io/repos/WoLpH/portalocker/badge.svg?branch=master + :alt: Coverage Status + :target: https://coveralls.io/r/WoLpH/portalocker?branch=master + +Overview +-------- + +Portalocker is a library to provide an easy API to file locking. + +An important detail to note is that on Linux and Unix systems the locks are +advisory by default. By specifying the `-o mand` option to the mount command it +is possible to enable mandatory file locking on Linux. This is generally not +recommended however. For more information about the subject: + + - https://en.wikipedia.org/wiki/File_locking + - http://stackoverflow.com/questions/39292051/portalocker-does-not-seem-to-lock + - https://stackoverflow.com/questions/12062466/mandatory-file-lock-on-linux + +The module is currently maintained by Rick van Hattem <[email protected]>. +The project resides at https://github.com/WoLpH/portalocker . Bugs and feature +requests can be submitted there. Patches are also very welcome. + +Tips +---- + +On some networked filesystems it might be needed to force a `os.fsync()` before +closing the file so it's actually written before another client reads the file. +Effectively this comes down to: + +:: + + with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: + # do what you need to do + ... + + # flush and sync to filesystem + fh.flush() + os.fsync(fh.fileno()) + +Links +----- + +* Documentation + - http://portalocker.readthedocs.org/en/latest/ +* Source + - https://github.com/WoLpH/portalocker +* Bug reports + - https://github.com/WoLpH/portalocker/issues +* Package homepage + - https://pypi.python.org/pypi/portalocker +* My blog + - http://w.wol.ph/ + +Examples +-------- + +To make sure your cache generation scripts don't race, use the `Lock` class: + +>>> import portalocker +>>> with portalocker.Lock('somefile', timeout=1) as fh: + print >>fh, 'writing some stuff to my cache...' + +To customize the opening and locking a manual approach is also possible: + +>>> import portalocker +>>> file = open('somefile', 'r+') +>>> portalocker.lock(file, portalocker.LOCK_EX) +>>> file.seek(12) +>>> file.write('foo') +>>> file.close() + +Explicitly unlocking might not be needed in all cases: +https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 + +But can be done through: + +>>> portalocker.unlock(file) + +Do note that your data might still be in a buffer so it is possible that your +data is not available until you `flush()` or `close()`. + +More examples can be found in the +`tests <http://portalocker.readthedocs.io/en/latest/_modules/tests/tests.html>`_. + +Changelog +--------- + +See the `changelog <http://portalocker.readthedocs.io/en/latest/changelog.html>`_ page. + +License +------- + +See the `LICENSE <https://github.com/WoLpH/portalocker/blob/develop/LICENSE>`_ file. + diff --git a/contrib/python/portalocker/py2/portalocker/__about__.py b/contrib/python/portalocker/py2/portalocker/__about__.py new file mode 100644 index 00000000000..f16fe0cdf7d --- /dev/null +++ b/contrib/python/portalocker/py2/portalocker/__about__.py @@ -0,0 +1,7 @@ +__package_name__ = 'portalocker' +__author__ = 'Rick van Hattem' +__email__ = '[email protected]' +__version__ = '1.7.1' +__description__ = '''Wraps the portalocker recipe for easy usage''' +__url__ = 'https://github.com/WoLpH/portalocker' + diff --git a/contrib/python/portalocker/py2/portalocker/__init__.py b/contrib/python/portalocker/py2/portalocker/__init__.py new file mode 100644 index 00000000000..9bf27fee0fc --- /dev/null +++ b/contrib/python/portalocker/py2/portalocker/__init__.py @@ -0,0 +1,67 @@ +from . import __about__ +from . import constants +from . import exceptions +from . import portalocker +from . import utils + +#: The package name on Pypi +__package_name__ = __about__.__package_name__ +#: Current author and maintainer, view the git history for the previous ones +__author__ = __about__.__author__ +#: Current author's email address +__email__ = __about__.__email__ +#: Version number +__version__ = '1.7.1' +#: Package description for Pypi +__description__ = __about__.__description__ +#: Package homepage +__url__ = __about__.__url__ + + +#: Exception thrown when the file is already locked by someone else +AlreadyLocked = exceptions.AlreadyLocked +#: Exception thrown if an error occurred during locking +LockException = exceptions.LockException + + +#: Lock a file. Note that this is an advisory lock on Linux/Unix systems +lock = portalocker.lock +#: Unlock a file +unlock = portalocker.unlock + +#: Place an exclusive lock. +#: Only one process may hold an exclusive lock for a given file at a given +#: time. +LOCK_EX = constants.LOCK_EX + +#: Place a shared lock. +#: More than one process may hold a shared lock for a given file at a given +#: time. +LOCK_SH = constants.LOCK_SH + +#: Acquire the lock in a non-blocking fashion. +LOCK_NB = constants.LOCK_NB + +#: Remove an existing lock held by this process. +LOCK_UN = constants.LOCK_UN + +#: Locking utility class to automatically handle opening with timeouts and +#: context wrappers +Lock = utils.Lock +RLock = utils.RLock +TemporaryFileLock = utils.TemporaryFileLock +open_atomic = utils.open_atomic + +__all__ = [ + 'lock', + 'unlock', + 'LOCK_EX', + 'LOCK_SH', + 'LOCK_NB', + 'LOCK_UN', + 'LockException', + 'Lock', + 'AlreadyLocked', + 'open_atomic', +] + diff --git a/contrib/python/portalocker/py2/portalocker/constants.py b/contrib/python/portalocker/py2/portalocker/constants.py new file mode 100644 index 00000000000..fb0927e2da9 --- /dev/null +++ b/contrib/python/portalocker/py2/portalocker/constants.py @@ -0,0 +1,39 @@ +''' +Locking constants + +Lock types: + +- `LOCK_EX` exclusive lock +- `LOCK_SH` shared lock + +Lock flags: + +- `LOCK_NB` non-blocking + +Manually unlock, only needed internally + +- `LOCK_UN` unlock +''' +import os + +# The actual tests will execute the code anyhow so the following code can +# safely be ignored from the coverage tests +if os.name == 'nt': # pragma: no cover + import msvcrt + + LOCK_EX = 0x1 #: exclusive lock + LOCK_SH = 0x2 #: shared lock + LOCK_NB = 0x4 #: non-blocking + LOCK_UN = msvcrt.LK_UNLCK #: unlock + +elif os.name == 'posix': # pragma: no cover + import fcntl + + LOCK_EX = fcntl.LOCK_EX #: exclusive lock + LOCK_SH = fcntl.LOCK_SH #: shared lock + LOCK_NB = fcntl.LOCK_NB #: non-blocking + LOCK_UN = fcntl.LOCK_UN #: unlock + +else: # pragma: no cover + raise RuntimeError('PortaLocker only defined for nt and posix platforms') + diff --git a/contrib/python/portalocker/py2/portalocker/exceptions.py b/contrib/python/portalocker/py2/portalocker/exceptions.py new file mode 100644 index 00000000000..bb2b35eb7bc --- /dev/null +++ b/contrib/python/portalocker/py2/portalocker/exceptions.py @@ -0,0 +1,19 @@ +class BaseLockException(Exception): + # Error codes: + LOCK_FAILED = 1 + + def __init__(self, *args, **kwargs): + self.fh = kwargs.pop('fh', None) + Exception.__init__(self, *args, **kwargs) + + +class LockException(BaseLockException): + pass + + +class AlreadyLocked(BaseLockException): + pass + + +class FileToLarge(BaseLockException): + pass diff --git a/contrib/python/portalocker/py2/portalocker/portalocker.py b/contrib/python/portalocker/py2/portalocker/portalocker.py new file mode 100644 index 00000000000..460cf06e476 --- /dev/null +++ b/contrib/python/portalocker/py2/portalocker/portalocker.py @@ -0,0 +1,148 @@ +import os +import sys +from . import exceptions +from . import constants + + +if os.name == 'nt': # pragma: no cover + import win32con + import win32file + import pywintypes + import winerror + import msvcrt + __overlapped = pywintypes.OVERLAPPED() + + if sys.version_info.major == 2: + lock_length = -1 + else: + lock_length = int(2**31 - 1) + + def lock(file_, flags): + if flags & constants.LOCK_SH: + if sys.version_info.major == 2: + if flags & constants.LOCK_NB: + mode = win32con.LOCKFILE_FAIL_IMMEDIATELY + else: + mode = 0 + + else: + if flags & constants.LOCK_NB: + mode = msvcrt.LK_NBRLCK + else: + mode = msvcrt.LK_RLCK + + # is there any reason not to reuse the following structure? + hfile = win32file._get_osfhandle(file_.fileno()) + try: + win32file.LockFileEx(hfile, mode, 0, -0x10000, __overlapped) + except pywintypes.error as exc_value: + # error: (33, 'LockFileEx', 'The process cannot access the file + # because another process has locked a portion of the file.') + if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + exc_value.strerror, + fh=file_) + else: + # Q: Are there exceptions/codes we should be dealing with + # here? + raise + else: + mode = win32con.LOCKFILE_EXCLUSIVE_LOCK + if flags & constants.LOCK_NB: + mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY + + if flags & constants.LOCK_NB: + mode = msvcrt.LK_NBLCK + else: + mode = msvcrt.LK_LOCK + + # windows locks byte ranges, so make sure to lock from file start + try: + savepos = file_.tell() + if savepos: + # [ ] test exclusive lock fails on seek here + # [ ] test if shared lock passes this point + file_.seek(0) + # [x] check if 0 param locks entire file (not documented in + # Python) + # [x] fails with "IOError: [Errno 13] Permission denied", + # but -1 seems to do the trick + + try: + msvcrt.locking(file_.fileno(), mode, lock_length) + except IOError as exc_value: + # [ ] be more specific here + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + exc_value.strerror, + fh=file_) + finally: + if savepos: + file_.seek(savepos) + except IOError as exc_value: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, exc_value.strerror, + fh=file_) + + def unlock(file_): + try: + savepos = file_.tell() + if savepos: + file_.seek(0) + + try: + msvcrt.locking(file_.fileno(), constants.LOCK_UN, lock_length) + except IOError as exc_value: + if exc_value.strerror == 'Permission denied': + hfile = win32file._get_osfhandle(file_.fileno()) + try: + win32file.UnlockFileEx( + hfile, 0, -0x10000, __overlapped) + except pywintypes.error as exc_value: + if exc_value.winerror == winerror.ERROR_NOT_LOCKED: + # error: (158, 'UnlockFileEx', + # 'The segment is already unlocked.') + # To match the 'posix' implementation, silently + # ignore this error + pass + else: + # Q: Are there exceptions/codes we should be + # dealing with here? + raise + else: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + exc_value.strerror, + fh=file_) + finally: + if savepos: + file_.seek(savepos) + except IOError as exc_value: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, exc_value.strerror, + fh=file_) + +elif os.name == 'posix': # pragma: no cover + import fcntl + + def lock(file_, flags): + locking_exceptions = IOError, + try: # pragma: no cover + locking_exceptions += BlockingIOError, + except NameError: # pragma: no cover + pass + + try: + fcntl.flock(file_.fileno(), flags) + except locking_exceptions as exc_value: + # The exception code varies on different systems so we'll catch + # every IO error + raise exceptions.LockException(exc_value, fh=file_) + + def unlock(file_): + fcntl.flock(file_.fileno(), constants.LOCK_UN) + +else: # pragma: no cover + raise RuntimeError('PortaLocker only defined for nt and posix platforms') + diff --git a/contrib/python/portalocker/py2/portalocker/utils.py b/contrib/python/portalocker/py2/portalocker/utils.py new file mode 100644 index 00000000000..8baebc2b200 --- /dev/null +++ b/contrib/python/portalocker/py2/portalocker/utils.py @@ -0,0 +1,256 @@ +import os +import time +import atexit +import tempfile +import contextlib +from . import exceptions +from . import constants +from . import portalocker + +current_time = getattr(time, "monotonic", time.time) + +DEFAULT_TIMEOUT = 5 +DEFAULT_CHECK_INTERVAL = 0.25 +LOCK_METHOD = constants.LOCK_EX | constants.LOCK_NB + +__all__ = [ + 'Lock', + 'open_atomic', +] + + +def open_atomic(filename, binary=True): + '''Open a file for atomic writing. Instead of locking this method allows + you to write the entire file and move it to the actual location. Note that + this makes the assumption that a rename is atomic on your platform which + is generally the case but not a guarantee. + + http://docs.python.org/library/os.html#os.rename + + >>> filename = 'test_file.txt' + >>> if os.path.exists(filename): + ... os.remove(filename) + + >>> with open_atomic(filename) as fh: + ... written = fh.write(b'test') + >>> assert os.path.exists(filename) + >>> os.remove(filename) + + ''' + assert not os.path.exists(filename), '%r exists' % filename + path, name = os.path.split(filename) + + # Create the parent directory if it doesn't exist + if path and not os.path.isdir(path): # pragma: no cover + os.makedirs(path) + + temp_fh = tempfile.NamedTemporaryFile( + mode=binary and 'wb' or 'w', + dir=path, + delete=False, + ) + yield temp_fh + temp_fh.flush() + os.fsync(temp_fh.fileno()) + temp_fh.close() + try: + os.rename(temp_fh.name, filename) + finally: + try: + os.remove(temp_fh.name) + except Exception: + pass + + +class Lock(object): + + def __init__( + self, filename, mode='a', timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, + flags=LOCK_METHOD, **file_open_kwargs): + '''Lock manager with build-in timeout + + filename -- filename + mode -- the open mode, 'a' or 'ab' should be used for writing + truncate -- use truncate to emulate 'w' mode, None is disabled, 0 is + truncate to 0 bytes + timeout -- timeout when trying to acquire a lock + check_interval -- check interval while waiting + fail_when_locked -- after the initial lock failed, return an error + or lock the file + **file_open_kwargs -- The kwargs for the `open(...)` call + + fail_when_locked is useful when multiple threads/processes can race + when creating a file. If set to true than the system will wait till + the lock was acquired and then return an AlreadyLocked exception. + + Note that the file is opened first and locked later. So using 'w' as + mode will result in truncate _BEFORE_ the lock is checked. + ''' + + if 'w' in mode: + truncate = True + mode = mode.replace('w', 'a') + else: + truncate = False + + self.fh = None + self.filename = filename + self.mode = mode + self.truncate = truncate + self.timeout = timeout + self.check_interval = check_interval + self.fail_when_locked = fail_when_locked + self.flags = flags + self.file_open_kwargs = file_open_kwargs + + def acquire( + self, timeout=None, check_interval=None, fail_when_locked=None): + '''Acquire the locked filehandle''' + if timeout is None: + timeout = self.timeout + if timeout is None: + timeout = 0 + + if check_interval is None: + check_interval = self.check_interval + + if fail_when_locked is None: + fail_when_locked = self.fail_when_locked + + # If we already have a filehandle, return it + fh = self.fh + if fh: + return fh + + # Get a new filehandler + fh = self._get_fh() + try: + # Try to lock + fh = self._get_lock(fh) + except exceptions.LockException as exception: + # Try till the timeout has passed + timeoutend = current_time() + timeout + while timeoutend > current_time(): + # Wait a bit + time.sleep(check_interval) + + # Try again + try: + + # We already tried to the get the lock + # If fail_when_locked is true, then stop trying + if fail_when_locked: + raise exceptions.AlreadyLocked(exception) + + else: # pragma: no cover + # We've got the lock + fh = self._get_lock(fh) + break + + except exceptions.LockException: + pass + + else: + fh.close() + # We got a timeout... reraising + raise exceptions.LockException(exception) + + # Prepare the filehandle (truncate if needed) + fh = self._prepare_fh(fh) + + self.fh = fh + return fh + + def release(self): + '''Releases the currently locked file handle''' + if self.fh: + portalocker.unlock(self.fh) + self.fh.close() + self.fh = None + + def _get_fh(self): + '''Get a new filehandle''' + return open(self.filename, self.mode, **self.file_open_kwargs) + + def _get_lock(self, fh): + ''' + Try to lock the given filehandle + + returns LockException if it fails''' + portalocker.lock(fh, self.flags) + return fh + + def _prepare_fh(self, fh): + ''' + Prepare the filehandle for usage + + If truncate is a number, the file will be truncated to that amount of + bytes + ''' + if self.truncate: + fh.seek(0) + fh.truncate(0) + + return fh + + def __enter__(self): + return self.acquire() + + def __exit__(self, type_, value, tb): + self.release() + + def __delete__(self, instance): # pragma: no cover + instance.release() + + +class RLock(Lock): + """ + A reentrant lock, functions in a similar way to threading.RLock in that it + can be acquired multiple times. When the corresponding number of release() + calls are made the lock will finally release the underlying file lock. + """ + def __init__( + self, filename, mode='a', timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, + flags=LOCK_METHOD): + super(RLock, self).__init__(filename, mode, timeout, check_interval, + fail_when_locked, flags) + self._acquire_count = 0 + + def acquire( + self, timeout=None, check_interval=None, fail_when_locked=None): + if self._acquire_count >= 1: + fh = self.fh + else: + fh = super(RLock, self).acquire(timeout, check_interval, + fail_when_locked) + self._acquire_count += 1 + return fh + + def release(self): + if self._acquire_count == 0: + raise exceptions.LockException( + "Cannot release more times than acquired") + + if self._acquire_count == 1: + super(RLock, self).release() + self._acquire_count -= 1 + + +class TemporaryFileLock(Lock): + + def __init__(self, filename='.lock', timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=True, + flags=LOCK_METHOD): + + Lock.__init__(self, filename=filename, mode='w', timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, flags=flags) + atexit.register(self.release) + + def release(self): + Lock.release(self) + if os.path.isfile(self.filename): # pragma: no branch + os.unlink(self.filename) diff --git a/contrib/python/portalocker/py2/tests/__init__.py b/contrib/python/portalocker/py2/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/portalocker/py2/tests/__init__.py diff --git a/contrib/python/portalocker/py2/tests/conftest.py b/contrib/python/portalocker/py2/tests/conftest.py new file mode 100644 index 00000000000..a92117e3951 --- /dev/null +++ b/contrib/python/portalocker/py2/tests/conftest.py @@ -0,0 +1,14 @@ +import py +import pytest + + +def tmpfile(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp('temp') + filename = tmpdir.join('tmpfile') + yield str(filename) + try: + filename.remove(ignore_errors=True) + except (py.error.EBUSY, py.error.ENOENT): + pass + diff --git a/contrib/python/portalocker/py2/tests/temporary_file_lock.py b/contrib/python/portalocker/py2/tests/temporary_file_lock.py new file mode 100644 index 00000000000..b250bad6510 --- /dev/null +++ b/contrib/python/portalocker/py2/tests/temporary_file_lock.py @@ -0,0 +1,14 @@ +import os +import portalocker + + +def test_temporary_file_lock(tmpfile): + with portalocker.TemporaryFileLock(tmpfile): + pass + + assert not os.path.isfile(tmpfile) + + lock = portalocker.TemporaryFileLock(tmpfile) + lock.acquire() + del lock + diff --git a/contrib/python/portalocker/py2/tests/test_combined.py b/contrib/python/portalocker/py2/tests/test_combined.py new file mode 100644 index 00000000000..594de74b9ba --- /dev/null +++ b/contrib/python/portalocker/py2/tests/test_combined.py @@ -0,0 +1,15 @@ +import sys + + +def test_combined(tmpdir): + from distutils import dist + import setup + + output_file = tmpdir.join('combined.py') + combine = setup.Combine(dist.Distribution()) + combine.output_file = str(output_file) + combine.run() + sys.path.append(output_file.dirname) + import combined + assert combined + diff --git a/contrib/python/portalocker/py2/tests/tests.py b/contrib/python/portalocker/py2/tests/tests.py new file mode 100644 index 00000000000..a6567426be0 --- /dev/null +++ b/contrib/python/portalocker/py2/tests/tests.py @@ -0,0 +1,202 @@ +from __future__ import print_function +from __future__ import with_statement + +import pytest +import portalocker + + +def test_exceptions(tmpfile): + # Open the file 2 times + a = open(tmpfile, 'a') + b = open(tmpfile, 'a') + + # Lock exclusive non-blocking + lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB + + # First lock file a + portalocker.lock(a, lock_flags) + + # Now see if we can lock file b + with pytest.raises(portalocker.LockException): + portalocker.lock(b, lock_flags) + + # Cleanup + a.close() + b.close() + + +def test_with_timeout(tmpfile): + # Open the file 2 times + with pytest.raises(portalocker.AlreadyLocked): + with portalocker.Lock(tmpfile, timeout=0.1) as fh: + print('writing some stuff to my cache...', file=fh) + with portalocker.Lock(tmpfile, timeout=0.1, mode='wb', + fail_when_locked=True): + pass + print('writing more stuff to my cache...', file=fh) + + +def test_without_timeout(tmpfile): + # Open the file 2 times + with pytest.raises(portalocker.LockException): + with portalocker.Lock(tmpfile, timeout=None) as fh: + print('writing some stuff to my cache...', file=fh) + with portalocker.Lock(tmpfile, timeout=None, mode='w'): + pass + print('writing more stuff to my cache...', file=fh) + + +def test_without_fail(tmpfile): + # Open the file 2 times + with pytest.raises(portalocker.LockException): + with portalocker.Lock(tmpfile, timeout=0.1) as fh: + print('writing some stuff to my cache...', file=fh) + lock = portalocker.Lock(tmpfile, timeout=0.1) + lock.acquire(check_interval=0.05, fail_when_locked=False) + + +def test_simple(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + fh = open(tmpfile, 'r+') + portalocker.lock(fh, portalocker.LOCK_EX) + + fh.seek(13) + fh.write('foo') + + # Make sure we didn't overwrite the original text + fh.seek(0) + assert fh.read(13) == 'spam and eggs' + + portalocker.unlock(fh) + fh.close() + + +def test_truncate(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + with portalocker.Lock(tmpfile, mode='a+') as fh: + # Make sure we didn't overwrite the original text + fh.seek(0) + assert fh.read(13) == 'spam and eggs' + + with portalocker.Lock(tmpfile, mode='w+') as fh: + # Make sure we truncated the file + assert fh.read() == '' + + +def test_class(tmpfile): + lock = portalocker.Lock(tmpfile) + lock2 = portalocker.Lock(tmpfile, fail_when_locked=False, timeout=0.01) + + with lock: + lock.acquire() + + with pytest.raises(portalocker.LockException): + with lock2: + pass + + with lock2: + pass + + +def test_acquire_release(tmpfile): + lock = portalocker.Lock(tmpfile) + lock2 = portalocker.Lock(tmpfile, fail_when_locked=False) + + lock.acquire() # acquire lock when nobody is using it + with pytest.raises(portalocker.LockException): + # another party should not be able to acquire the lock + lock2.acquire(timeout=0.01) + + # re-acquire a held lock is a no-op + lock.acquire() + + lock.release() # release the lock + lock.release() # second release does nothing + + +def test_rlock_acquire_release_count(tmpfile): + lock = portalocker.RLock(tmpfile) + # Twice acquire + h = lock.acquire() + assert not h.closed + lock.acquire() + assert not h.closed + + # Two release + lock.release() + assert not h.closed + lock.release() + assert h.closed + + +def test_rlock_acquire_release(tmpfile): + lock = portalocker.RLock(tmpfile) + lock2 = portalocker.RLock(tmpfile, fail_when_locked=False) + + lock.acquire() # acquire lock when nobody is using it + with pytest.raises(portalocker.LockException): + # another party should not be able to acquire the lock + lock2.acquire(timeout=0.01) + + # Now acquire again + lock.acquire() + + lock.release() # release the lock + lock.release() # second release does nothing + + +def test_release_unacquired(tmpfile): + with pytest.raises(portalocker.LockException): + portalocker.RLock(tmpfile).release() + + +def test_exlusive(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + fh = open(tmpfile, 'r') + portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) + + # Make sure we can't read the locked file + with pytest.raises(portalocker.LockException): + with open(tmpfile, 'r') as fh2: + portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) + fh2.read() + + # Make sure we can't write the locked file + with pytest.raises(portalocker.LockException): + with open(tmpfile, 'w+') as fh2: + portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) + fh2.write('surprise and fear') + + # Make sure we can explicitly unlock the file + portalocker.unlock(fh) + fh.close() + + +def test_shared(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + f = open(tmpfile, 'r') + portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB) + + # Make sure we can read the locked file + with open(tmpfile, 'r') as fh2: + portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB) + assert fh2.read() == 'spam and eggs' + + # Make sure we can't write the locked file + with pytest.raises(portalocker.LockException): + with open(tmpfile, 'w+') as fh2: + portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) + fh2.write('surprise and fear') + + # Make sure we can explicitly unlock the file + portalocker.unlock(f) + f.close() + diff --git a/contrib/python/portalocker/py2/tests/ya.make b/contrib/python/portalocker/py2/tests/ya.make new file mode 100644 index 00000000000..6917e170a2a --- /dev/null +++ b/contrib/python/portalocker/py2/tests/ya.make @@ -0,0 +1,20 @@ +PY2TEST() + +SUBSCRIBER(g:python-contrib) + +NO_LINT() + +PEERDIR( + contrib/python/portalocker +) + +TEST_SRCS( + __init__.py + conftest.py + temporary_file_lock.py + # Tests intallation. + # test_combined.py + tests.py +) + +END() diff --git a/contrib/python/portalocker/py2/ya.make b/contrib/python/portalocker/py2/ya.make new file mode 100644 index 00000000000..bad9595eaf9 --- /dev/null +++ b/contrib/python/portalocker/py2/ya.make @@ -0,0 +1,33 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +SUBSCRIBER(g:python-contrib) + +VERSION(1.7.1) + +LICENSE(PSF-2.0) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + portalocker/__about__.py + portalocker/__init__.py + portalocker/constants.py + portalocker/exceptions.py + portalocker/portalocker.py + portalocker/utils.py +) + +RESOURCE_FILES( + PREFIX contrib/python/portalocker/py2/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() + +RECURSE( + tests +) diff --git a/contrib/python/portalocker/py3/.dist-info/METADATA b/contrib/python/portalocker/py3/.dist-info/METADATA new file mode 100644 index 00000000000..8202928049b --- /dev/null +++ b/contrib/python/portalocker/py3/.dist-info/METADATA @@ -0,0 +1,259 @@ +Metadata-Version: 2.4 +Name: portalocker +Version: 3.2.0 +Summary: Wraps the portalocker recipe for easy usage +Author-email: Rick van Hattem <[email protected]> +License-Expression: BSD-3-Clause +Project-URL: bugs, https://github.com/wolph/portalocker/issues +Project-URL: documentation, https://portalocker.readthedocs.io/en/latest/ +Project-URL: repository, https://github.com/wolph/portalocker/ +Keywords: locking,locks,with,statement,windows,linux,unix +Platform: any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Development Status :: 6 - Mature +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: MS-DOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: Microsoft +Classifier: Operating System :: POSIX :: BSD :: FreeBSD +Classifier: Operating System :: POSIX :: BSD +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: POSIX :: SunOS/Solaris +Classifier: Operating System :: POSIX +Classifier: Operating System :: Unix +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: IronPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: Implementation +Classifier: Programming Language :: Python +Classifier: Topic :: Education :: Testing +Classifier: Topic :: Office/Business +Classifier: Topic :: Other/Nonlisted Topic +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: System :: Monitoring +Classifier: Typing :: Typed +Requires-Python: >=3.9 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: pywin32>=226; platform_system == "Windows" +Provides-Extra: docs +Requires-Dist: portalocker[tests]; extra == "docs" +Provides-Extra: tests +Requires-Dist: coverage-conditional-plugin>=0.9.0; extra == "tests" +Requires-Dist: portalocker[redis]; extra == "tests" +Requires-Dist: pytest-cov>=2.8.1; extra == "tests" +Requires-Dist: pytest-mypy>=0.8.0; extra == "tests" +Requires-Dist: pytest-rerunfailures>=15.0; extra == "tests" +Requires-Dist: pytest-timeout>=2.1.0; extra == "tests" +Requires-Dist: pytest>=5.4.1; extra == "tests" +Requires-Dist: sphinx>=6.0.0; extra == "tests" +Requires-Dist: types-pywin32>=310.0.0.20250429; extra == "tests" +Requires-Dist: types-redis; extra == "tests" +Provides-Extra: redis +Requires-Dist: redis; extra == "redis" +Dynamic: license-file + +############################################ +portalocker - Cross-platform locking library +############################################ + +.. image:: https://github.com/WoLpH/portalocker/actions/workflows/python-package.yml/badge.svg?branch=master + :alt: Linux Test Status + :target: https://github.com/WoLpH/portalocker/actions/ + +.. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true + :alt: Windows Tests Status + :target: https://ci.appveyor.com/project/WoLpH/portalocker + +.. image:: https://coveralls.io/repos/WoLpH/portalocker/badge.svg?branch=master + :alt: Coverage Status + :target: https://coveralls.io/r/WoLpH/portalocker?branch=master + +Overview +-------- + +Portalocker is a library to provide an easy API to file locking. + +An important detail to note is that on Linux and Unix systems the locks are +advisory by default. By specifying the `-o mand` option to the mount command it +is possible to enable mandatory file locking on Linux. This is generally not +recommended however. For more information about the subject: + + - https://en.wikipedia.org/wiki/File_locking + - http://stackoverflow.com/questions/39292051/portalocker-does-not-seem-to-lock + - https://stackoverflow.com/questions/12062466/mandatory-file-lock-on-linux + +The module is currently maintained by Rick van Hattem <[email protected]>. +The project resides at https://github.com/WoLpH/portalocker . Bugs and feature +requests can be submitted there. Patches are also very welcome. + +Security contact information +------------------------------------------------------------------------------ + +To report a security vulnerability, please use the +`Tidelift security contact <https://tidelift.com/security>`_. +Tidelift will coordinate the fix and disclosure. + +Redis Locks +----------- + +This library now features a lock based on Redis which allows for locks across +multiple threads, processes and even distributed locks across multiple +computers. + +It is an extremely reliable Redis lock that is based on pubsub. + +As opposed to most Redis locking systems based on key/value pairs, +this locking method is based on the pubsub system. The big advantage is +that if the connection gets killed due to network issues, crashing +processes or otherwise, it will still immediately unlock instead of +waiting for a lock timeout. + +First make sure you have everything installed correctly: + +:: + + pip install "portalocker[redis]" + +Usage is really easy: + +:: + + import portalocker + + lock = portalocker.RedisLock('some_lock_channel_name') + + with lock: + print('do something here') + +The API is essentially identical to the other ``Lock`` classes so in addition +to the ``with`` statement you can also use ``lock.acquire(...)``. + +Python 2 +-------- + +Python 2 was supported in versions before Portalocker 2.0. If you are still +using +Python 2, +you can run this to install: + +:: + + pip install "portalocker<2" + +Tips +---- + +On some networked filesystems it might be needed to force a `os.fsync()` before +closing the file so it's actually written before another client reads the file. +Effectively this comes down to: + +:: + + with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: + # do what you need to do + ... + + # flush and sync to filesystem + fh.flush() + os.fsync(fh.fileno()) + +Links +----- + +* Documentation + - http://portalocker.readthedocs.org/en/latest/ +* Source + - https://github.com/WoLpH/portalocker +* Bug reports + - https://github.com/WoLpH/portalocker/issues +* Package homepage + - https://pypi.python.org/pypi/portalocker +* My blog + - http://w.wol.ph/ + +Examples +-------- + +To make sure your cache generation scripts don't race, use the `Lock` class: + +>>> import portalocker +>>> with portalocker.Lock('somefile', timeout=1) as fh: +... print('writing some stuff to my cache...', file=fh) + +To customize the opening and locking a manual approach is also possible: + +>>> import portalocker +>>> file = open('somefile', 'r+') +>>> portalocker.lock(file, portalocker.LockFlags.EXCLUSIVE) +>>> file.seek(12) +>>> file.write('foo') +>>> file.close() + +Explicitly unlocking is not needed in most cases but omitting it has been known +to cause issues: +https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 + +If needed, it can be done through: + +>>> portalocker.unlock(file) + +Do note that your data might still be in a buffer so it is possible that your +data is not available until you `flush()` or `close()`. + +To create a cross platform bounded semaphore across multiple processes you can +use the `BoundedSemaphore` class which functions somewhat similar to +`threading.BoundedSemaphore`: + +>>> import portalocker +>>> n = 2 +>>> timeout = 0.1 + +>>> semaphore_a = portalocker.BoundedSemaphore(n, timeout=timeout) +>>> semaphore_b = portalocker.BoundedSemaphore(n, timeout=timeout) +>>> semaphore_c = portalocker.BoundedSemaphore(n, timeout=timeout) + +>>> semaphore_a.acquire() +<portalocker.utils.Lock object at ...> +>>> semaphore_b.acquire() +<portalocker.utils.Lock object at ...> +>>> semaphore_c.acquire() +Traceback (most recent call last): + ... +portalocker.exceptions.AlreadyLocked + + +More examples can be found in the +`tests <http://portalocker.readthedocs.io/en/latest/_modules/tests/tests.html>`_. + + +Versioning +---------- + +This library follows `Semantic Versioning <http://semver.org/>`_. + + +Changelog +--------- + +Every release has a ``git tag`` with a commit message for the tag +explaining what was added and/or changed. The list of tags/releases +including the commit messages can be found here: +https://github.com/WoLpH/portalocker/releases + +License +------- + +See the `LICENSE <https://github.com/WoLpH/portalocker/blob/develop/LICENSE>`_ file. + diff --git a/contrib/python/portalocker/py3/.dist-info/top_level.txt b/contrib/python/portalocker/py3/.dist-info/top_level.txt new file mode 100644 index 00000000000..7bbc14e6fa6 --- /dev/null +++ b/contrib/python/portalocker/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +portalocker diff --git a/contrib/python/portalocker/py3/.yandex_meta/yamaker.yaml b/contrib/python/portalocker/py3/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..bca0c4e5337 --- /dev/null +++ b/contrib/python/portalocker/py3/.yandex_meta/yamaker.yaml @@ -0,0 +1,4 @@ +keep: + - tests/* +exclude: + - portalocker_tests/* diff --git a/contrib/python/portalocker/py3/LICENSE b/contrib/python/portalocker/py3/LICENSE new file mode 100644 index 00000000000..b638bda0d3b --- /dev/null +++ b/contrib/python/portalocker/py3/LICENSE @@ -0,0 +1,11 @@ +Copyright 2022 Rick van Hattem + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/portalocker/py3/README.rst b/contrib/python/portalocker/py3/README.rst new file mode 100644 index 00000000000..c5ef42f614b --- /dev/null +++ b/contrib/python/portalocker/py3/README.rst @@ -0,0 +1,193 @@ +############################################ +portalocker - Cross-platform locking library +############################################ + +.. image:: https://github.com/WoLpH/portalocker/actions/workflows/python-package.yml/badge.svg?branch=master + :alt: Linux Test Status + :target: https://github.com/WoLpH/portalocker/actions/ + +.. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true + :alt: Windows Tests Status + :target: https://ci.appveyor.com/project/WoLpH/portalocker + +.. image:: https://coveralls.io/repos/WoLpH/portalocker/badge.svg?branch=master + :alt: Coverage Status + :target: https://coveralls.io/r/WoLpH/portalocker?branch=master + +Overview +-------- + +Portalocker is a library to provide an easy API to file locking. + +An important detail to note is that on Linux and Unix systems the locks are +advisory by default. By specifying the `-o mand` option to the mount command it +is possible to enable mandatory file locking on Linux. This is generally not +recommended however. For more information about the subject: + + - https://en.wikipedia.org/wiki/File_locking + - http://stackoverflow.com/questions/39292051/portalocker-does-not-seem-to-lock + - https://stackoverflow.com/questions/12062466/mandatory-file-lock-on-linux + +The module is currently maintained by Rick van Hattem <[email protected]>. +The project resides at https://github.com/WoLpH/portalocker . Bugs and feature +requests can be submitted there. Patches are also very welcome. + +Security contact information +------------------------------------------------------------------------------ + +To report a security vulnerability, please use the +`Tidelift security contact <https://tidelift.com/security>`_. +Tidelift will coordinate the fix and disclosure. + +Redis Locks +----------- + +This library now features a lock based on Redis which allows for locks across +multiple threads, processes and even distributed locks across multiple +computers. + +It is an extremely reliable Redis lock that is based on pubsub. + +As opposed to most Redis locking systems based on key/value pairs, +this locking method is based on the pubsub system. The big advantage is +that if the connection gets killed due to network issues, crashing +processes or otherwise, it will still immediately unlock instead of +waiting for a lock timeout. + +First make sure you have everything installed correctly: + +:: + + pip install "portalocker[redis]" + +Usage is really easy: + +:: + + import portalocker + + lock = portalocker.RedisLock('some_lock_channel_name') + + with lock: + print('do something here') + +The API is essentially identical to the other ``Lock`` classes so in addition +to the ``with`` statement you can also use ``lock.acquire(...)``. + +Python 2 +-------- + +Python 2 was supported in versions before Portalocker 2.0. If you are still +using +Python 2, +you can run this to install: + +:: + + pip install "portalocker<2" + +Tips +---- + +On some networked filesystems it might be needed to force a `os.fsync()` before +closing the file so it's actually written before another client reads the file. +Effectively this comes down to: + +:: + + with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: + # do what you need to do + ... + + # flush and sync to filesystem + fh.flush() + os.fsync(fh.fileno()) + +Links +----- + +* Documentation + - http://portalocker.readthedocs.org/en/latest/ +* Source + - https://github.com/WoLpH/portalocker +* Bug reports + - https://github.com/WoLpH/portalocker/issues +* Package homepage + - https://pypi.python.org/pypi/portalocker +* My blog + - http://w.wol.ph/ + +Examples +-------- + +To make sure your cache generation scripts don't race, use the `Lock` class: + +>>> import portalocker +>>> with portalocker.Lock('somefile', timeout=1) as fh: +... print('writing some stuff to my cache...', file=fh) + +To customize the opening and locking a manual approach is also possible: + +>>> import portalocker +>>> file = open('somefile', 'r+') +>>> portalocker.lock(file, portalocker.LockFlags.EXCLUSIVE) +>>> file.seek(12) +>>> file.write('foo') +>>> file.close() + +Explicitly unlocking is not needed in most cases but omitting it has been known +to cause issues: +https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 + +If needed, it can be done through: + +>>> portalocker.unlock(file) + +Do note that your data might still be in a buffer so it is possible that your +data is not available until you `flush()` or `close()`. + +To create a cross platform bounded semaphore across multiple processes you can +use the `BoundedSemaphore` class which functions somewhat similar to +`threading.BoundedSemaphore`: + +>>> import portalocker +>>> n = 2 +>>> timeout = 0.1 + +>>> semaphore_a = portalocker.BoundedSemaphore(n, timeout=timeout) +>>> semaphore_b = portalocker.BoundedSemaphore(n, timeout=timeout) +>>> semaphore_c = portalocker.BoundedSemaphore(n, timeout=timeout) + +>>> semaphore_a.acquire() +<portalocker.utils.Lock object at ...> +>>> semaphore_b.acquire() +<portalocker.utils.Lock object at ...> +>>> semaphore_c.acquire() +Traceback (most recent call last): + ... +portalocker.exceptions.AlreadyLocked + + +More examples can be found in the +`tests <http://portalocker.readthedocs.io/en/latest/_modules/tests/tests.html>`_. + + +Versioning +---------- + +This library follows `Semantic Versioning <http://semver.org/>`_. + + +Changelog +--------- + +Every release has a ``git tag`` with a commit message for the tag +explaining what was added and/or changed. The list of tags/releases +including the commit messages can be found here: +https://github.com/WoLpH/portalocker/releases + +License +------- + +See the `LICENSE <https://github.com/WoLpH/portalocker/blob/develop/LICENSE>`_ file. + diff --git a/contrib/python/portalocker/py3/portalocker/__about__.py b/contrib/python/portalocker/py3/portalocker/__about__.py new file mode 100644 index 00000000000..10a10202531 --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/__about__.py @@ -0,0 +1,6 @@ +__package_name__ = 'portalocker' +__author__ = 'Rick van Hattem' +__email__ = '[email protected]' +__version__ = '3.2.0' +__description__ = """Wraps the portalocker recipe for easy usage""" +__url__ = 'https://github.com/WoLpH/portalocker' diff --git a/contrib/python/portalocker/py3/portalocker/__init__.py b/contrib/python/portalocker/py3/portalocker/__init__.py new file mode 100644 index 00000000000..b1c451d2a51 --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/__init__.py @@ -0,0 +1,79 @@ +from . import __about__, constants, exceptions, portalocker +from .utils import ( + BoundedSemaphore, + Lock, + RLock, + TemporaryFileLock, + open_atomic, +) + +try: # pragma: no cover + from .redis import RedisLock +except ImportError: # pragma: no cover + RedisLock = None # type: ignore[assignment,misc] + + +#: The package name on Pypi +__package_name__ = __about__.__package_name__ +#: Current author and maintainer, view the git history for the previous ones +__author__ = __about__.__author__ +#: Current author's email address +__email__ = __about__.__email__ +#: Version number +__version__ = '3.2.0' +#: Package description for Pypi +__description__ = __about__.__description__ +#: Package homepage +__url__ = __about__.__url__ + + +#: Exception thrown when the file is already locked by someone else +AlreadyLocked = exceptions.AlreadyLocked +#: Exception thrown if an error occurred during locking +LockException = exceptions.LockException + + +#: Lock a file. Note that this is an advisory lock on Linux/Unix systems +lock = portalocker.lock +#: Unlock a file +unlock = portalocker.unlock + +#: Place an exclusive lock. +#: Only one process may hold an exclusive lock for a given file at a given +#: time. +LOCK_EX: constants.LockFlags = constants.LockFlags.EXCLUSIVE + +#: Place a shared lock. +#: More than one process may hold a shared lock for a given file at a given +#: time. +LOCK_SH: constants.LockFlags = constants.LockFlags.SHARED + +#: Acquire the lock in a non-blocking fashion. +LOCK_NB: constants.LockFlags = constants.LockFlags.NON_BLOCKING + +#: Remove an existing lock held by this process. +LOCK_UN: constants.LockFlags = constants.LockFlags.UNBLOCK + +#: Locking flags enum +LockFlags = constants.LockFlags + +#: Locking utility class to automatically handle opening with timeouts and +#: context wrappers + +__all__ = [ + 'LOCK_EX', + 'LOCK_NB', + 'LOCK_SH', + 'LOCK_UN', + 'AlreadyLocked', + 'BoundedSemaphore', + 'Lock', + 'LockException', + 'LockFlags', + 'RLock', + 'RedisLock', + 'TemporaryFileLock', + 'lock', + 'open_atomic', + 'unlock', +] diff --git a/contrib/python/portalocker/py3/portalocker/__main__.py b/contrib/python/portalocker/py3/portalocker/__main__.py new file mode 100644 index 00000000000..a573ad7931e --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/__main__.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import argparse +import logging +import pathlib +import re +import subprocess +import typing + +base_path = pathlib.Path(__file__).parent.parent +src_path = base_path / 'portalocker' +dist_path = base_path / 'dist' +_default_output_path = base_path / 'dist' / 'portalocker.py' + +_NAMES_RE = re.compile(r'(?P<names>[^()]+)$') +_RELATIVE_IMPORT_RE = re.compile( + r'^from \.(?P<from>.*?) import (?P<paren>\(?)(?P<names>[^()]+)$', +) +_USELESS_ASSIGNMENT_RE = re.compile(r'^(?P<name>\w+) = \1\n$') + +_TEXT_TEMPLATE = """''' +{} +''' + +""" + +logger = logging.getLogger(__name__) + + +def main(argv: typing.Sequence[str] | None = None) -> None: + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(required=True) + combine_parser = subparsers.add_parser( + 'combine', + help='Combine all Python files into a single unified `portalocker.py` ' + 'file for easy distribution', + ) + combine_parser.add_argument( + '--output-file', + '-o', + type=argparse.FileType('w'), + default=str(_default_output_path), + ) + + combine_parser.set_defaults(func=combine) + args = parser.parse_args(argv) + args.func(args) + + +def _read_file( # noqa: C901 + path: pathlib.Path, + seen_files: set[pathlib.Path], +) -> typing.Iterator[str]: + if path in seen_files: + return + + names: set[str] = set() + seen_files.add(path) + paren = False + from_ = None + try: + for line in path.open(encoding='ascii'): + if '__future__' in line: + continue + + if paren: + if ')' in line: + line = line.split(')', 1)[1] + paren = False + continue + + match = _NAMES_RE.match(line) + else: + match = _RELATIVE_IMPORT_RE.match(line) + + if match: + if not paren: + paren = bool(match.group('paren')) + from_ = match.group('from') + + if from_: + names.add(from_) + yield from _read_file(src_path / f'{from_}.py', seen_files) + else: + for name in match.group('names').split(','): + name = name.strip() + names.add(name) + yield from _read_file( + src_path / f'{name}.py', seen_files + ) + else: + yield _clean_line(line, names) + except UnicodeDecodeError as exception: # pragma: no cover + _, text, start_byte, end_byte, error = exception.args + + offset = 100 + snippet = text[start_byte - offset : end_byte + offset] + logger.error( # noqa: TRY400 + f'Invalid encoding for {path}: {error} at byte ' + f'({start_byte}:{end_byte})\n' + f'Snippet: {snippet!r}' + ) + raise + + +def _clean_line(line: str, names: set[str]) -> str: + # Replace `some_import.spam` with `spam` + if names: + joined_names = '|'.join(names) + line = re.sub(rf'\b({joined_names})\.', '', line) + + # Replace useless assignments (e.g. `spam = spam`) + return _USELESS_ASSIGNMENT_RE.sub('', line) + + +def combine(args: argparse.Namespace) -> None: + output_file = args.output_file + pathlib.Path(output_file.name).parent.mkdir(parents=True, exist_ok=True) + + # We're handling this separately because it has to be the first import. + output_file.write('from __future__ import annotations\n') + + output_file.write( + _TEXT_TEMPLATE.format( + (base_path / 'README.rst').read_text(encoding='ascii') + ), + ) + output_file.write( + _TEXT_TEMPLATE.format( + (base_path / 'LICENSE').read_text(encoding='ascii') + ), + ) + + seen_files: set[pathlib.Path] = set() + for line in _read_file(src_path / '__init__.py', seen_files): + output_file.write(line) + + output_file.flush() + output_file.close() + + logger.info(f'Wrote combined file to {output_file.name}') + # Run ruff if available. If not then just run the file. + try: # pragma: no cover + subprocess.run(['ruff', 'format', output_file.name], timeout=3) + subprocess.run( + ['ruff', 'check', '--fix', '--fix-only', output_file.name], + timeout=3, + ) + except FileNotFoundError: # pragma: no cover + logger.warning( + 'Ruff is not installed. Skipping linting and formatting step.' + ) + subprocess.run(['python3', output_file.name]) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/contrib/python/portalocker/py3/portalocker/constants.py b/contrib/python/portalocker/py3/portalocker/constants.py new file mode 100644 index 00000000000..5787725e727 --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/constants.py @@ -0,0 +1,59 @@ +""" +Locking constants + +Lock types: + +- `EXCLUSIVE` exclusive lock +- `SHARED` shared lock + +Lock flags: + +- `NON_BLOCKING` non-blocking + +Manually unlock, only needed internally + +- `UNBLOCK` unlock +""" + +import enum +import os + +# The actual tests will execute the code anyhow so the following code can +# safely be ignored from the coverage tests +if os.name == 'nt': # pragma: no cover + import msvcrt + + #: exclusive lock + LOCK_EX = 0x1 + #: shared lock + LOCK_SH = 0x2 + #: non-blocking + LOCK_NB = 0x4 + #: unlock + LOCK_UN = msvcrt.LK_UNLCK # type: ignore[attr-defined] + +elif os.name == 'posix': # pragma: no cover + import fcntl + + #: exclusive lock + LOCK_EX = fcntl.LOCK_EX # type: ignore[attr-defined] + #: shared lock + LOCK_SH = fcntl.LOCK_SH # type: ignore[attr-defined] + #: non-blocking + LOCK_NB = fcntl.LOCK_NB # type: ignore[attr-defined] + #: unlock + LOCK_UN = fcntl.LOCK_UN # type: ignore[attr-defined] + +else: # pragma: no cover + raise RuntimeError('PortaLocker only defined for nt and posix platforms') + + +class LockFlags(enum.IntFlag): + #: exclusive lock + EXCLUSIVE = LOCK_EX + #: shared lock + SHARED = LOCK_SH + #: non-blocking + NON_BLOCKING = LOCK_NB + #: unlock + UNBLOCK = LOCK_UN diff --git a/contrib/python/portalocker/py3/portalocker/exceptions.py b/contrib/python/portalocker/py3/portalocker/exceptions.py new file mode 100644 index 00000000000..00142b0e811 --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/exceptions.py @@ -0,0 +1,36 @@ +import typing + +from . import types + + +class BaseLockException(Exception): # noqa: N818 + # Error codes: + LOCK_FAILED: typing.Final = 1 + + strerror: typing.Optional[str] = None # ensure attribute always exists + + def __init__( + self, + *args: typing.Any, + fh: typing.Union[types.IO, None, int, types.HasFileno] = None, + **kwargs: typing.Any, + ) -> None: + self.fh = fh + self.strerror = ( + str(args[1]) + if len(args) > 1 and isinstance(args[1], str) + else None + ) + Exception.__init__(self, *args) + + +class LockException(BaseLockException): + pass + + +class AlreadyLocked(LockException): + pass + + +class FileToLarge(LockException): + pass diff --git a/contrib/python/portalocker/py3/portalocker/portalocker.py b/contrib/python/portalocker/py3/portalocker/portalocker.py new file mode 100644 index 00000000000..006cb57571e --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/portalocker.py @@ -0,0 +1,444 @@ +# pyright: reportUnknownMemberType=false, reportAttributeAccessIssue=false +"""Module portalocker. + +This module provides cross-platform file locking functionality. +The Windows implementation now supports two variants: + + 1. A default method using the Win32 API (win32file.LockFileEx/UnlockFileEx). + 2. An alternative that uses msvcrt.locking for exclusive locks (shared + locks still use the Win32 API). + +This version uses classes to encapsulate locking logic, while maintaining +the original external API, including the LOCKER constant for specific +backwards compatibility (POSIX) and Windows behavior. +""" + +import io +import os +import typing +from typing import ( + Any, + Callable, + Optional, + Union, + cast, +) + +from . import constants, exceptions, types + +# Alias for readability +LockFlags = constants.LockFlags + + +# Define a protocol for callable lockers +class LockCallable(typing.Protocol): + def __call__( + self, file_obj: types.FileArgument, flags: LockFlags + ) -> None: ... + + +class UnlockCallable(typing.Protocol): + def __call__(self, file_obj: types.FileArgument) -> None: ... + + +class BaseLocker: + """Base class for locker implementations.""" + + def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None: + """Lock the file.""" + raise NotImplementedError + + def unlock(self, file_obj: types.FileArgument) -> None: + """Unlock the file.""" + raise NotImplementedError + + +# Define refined LockerType with more specific types +LockerType = Union[ + # POSIX-style fcntl.flock callable + Callable[[Union[int, types.HasFileno], int], Any], + # Tuple of lock and unlock functions + tuple[LockCallable, UnlockCallable], + # BaseLocker instance + BaseLocker, + # BaseLocker class + type[BaseLocker], +] + +LOCKER: LockerType + +if os.name == 'nt': # pragma: not-posix + # Windows-specific helper functions + def _prepare_windows_file( + file_obj: types.FileArgument, + ) -> tuple[int, Optional[typing.IO[Any]], Optional[int]]: + """Prepare file for Windows: get fd, optionally seek and save pos.""" + if isinstance(file_obj, int): + # Plain file descriptor + return file_obj, None, None + + # Full IO objects (have tell/seek) -> preserve and restore position + if isinstance(file_obj, io.IOBase): + fd: int = file_obj.fileno() + original_pos = file_obj.tell() + if original_pos != 0: + file_obj.seek(0) + return fd, typing.cast(typing.IO[Any], file_obj), original_pos + # cast satisfies mypy: IOBase -> IO[Any] + + # Fallback: an object that only implements fileno() (HasFileno) + fd = typing.cast(types.HasFileno, file_obj).fileno() # type: ignore[redundant-cast] + return fd, None, None + + def _restore_windows_file_pos( + file_io_obj: Optional[typing.IO[Any]], + original_pos: Optional[int], + ) -> None: + """Restore file position if it was an IO object and pos was saved.""" + if file_io_obj and original_pos is not None and original_pos != 0: + file_io_obj.seek(original_pos) + + class Win32Locker(BaseLocker): + """Locker using Win32 API (LockFileEx/UnlockFileEx).""" + + _overlapped: Any # pywintypes.OVERLAPPED + _lock_bytes_low: int = -0x10000 + + def __init__(self) -> None: + try: + import pywintypes + except ImportError as e: + raise ImportError( + 'pywintypes is required for Win32Locker but not ' + 'found. Please install pywin32.' + ) from e + self._overlapped = pywintypes.OVERLAPPED() + + def _get_os_handle(self, fd: int) -> int: + try: + import msvcrt + except ImportError as e: + raise ImportError( + 'msvcrt is required for _get_os_handle on Windows ' + 'but not found.' + ) from e + return cast(int, msvcrt.get_osfhandle(fd)) # type: ignore[attr-defined,redundant-cast] + + def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None: + import pywintypes + import win32con + import win32file + import winerror + + fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) + os_fh = self._get_os_handle(fd) + + mode = 0 + if flags & LockFlags.NON_BLOCKING: + mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY + if flags & LockFlags.EXCLUSIVE: + mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK + + try: + win32file.LockFileEx( + os_fh, mode, 0, self._lock_bytes_low, self._overlapped + ) + except pywintypes.error as exc_value: # type: ignore[misc] + if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: + raise exceptions.AlreadyLocked( + exceptions.LockException.LOCK_FAILED, + exc_value.strerror, + fh=file_obj, # Pass original file_obj + ) from exc_value + else: + raise + finally: + _restore_windows_file_pos(io_obj_ctx, pos_ctx) + + def unlock(self, file_obj: types.FileArgument) -> None: + import pywintypes + import win32file + import winerror + + fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) + os_fh = self._get_os_handle(fd) + + try: + win32file.UnlockFileEx( + os_fh, 0, self._lock_bytes_low, self._overlapped + ) + except pywintypes.error as exc: # type: ignore[misc] + if exc.winerror != winerror.ERROR_NOT_LOCKED: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + exc.strerror, + fh=file_obj, # Pass original file_obj + ) from exc + except OSError as exc: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + exc.strerror, + fh=file_obj, # Pass original file_obj + ) from exc + finally: + _restore_windows_file_pos(io_obj_ctx, pos_ctx) + + class MsvcrtLocker(BaseLocker): + _win32_locker: Win32Locker + _msvcrt_lock_length: int = 0x10000 + + def __init__(self) -> None: + self._win32_locker = Win32Locker() + try: + import msvcrt + except ImportError as e: + raise ImportError( + 'msvcrt is required for MsvcrtLocker but not found.' + ) from e + + attrs = ['LK_LOCK', 'LK_RLCK', 'LK_NBLCK', 'LK_UNLCK', 'LK_NBRLCK'] + defaults = [0, 1, 2, 3, 2] # LK_NBRLCK often same as LK_NBLCK (2) + for attr, default_val in zip(attrs, defaults): + if not hasattr(msvcrt, attr): + setattr(msvcrt, attr, default_val) + + def lock(self, file_obj: types.FileArgument, flags: LockFlags) -> None: + import msvcrt + + if flags & LockFlags.SHARED: + win32_api_flags = LockFlags(0) + if flags & LockFlags.NON_BLOCKING: + win32_api_flags |= LockFlags.NON_BLOCKING + self._win32_locker.lock(file_obj, win32_api_flags) + return + + fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) + mode = ( + msvcrt.LK_NBLCK # type: ignore[attr-defined] + if flags & LockFlags.NON_BLOCKING + else msvcrt.LK_LOCK # type: ignore[attr-defined] + ) + + try: + msvcrt.locking( # type: ignore[attr-defined] + fd, + mode, + self._msvcrt_lock_length, + ) + except OSError as exc_value: + if exc_value.errno in (13, 16, 33, 36): + raise exceptions.AlreadyLocked( + exceptions.LockException.LOCK_FAILED, + str(exc_value), + fh=file_obj, # Pass original file_obj + ) from exc_value + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + str(exc_value), + fh=file_obj, # Pass original file_obj + ) from exc_value + finally: + _restore_windows_file_pos(io_obj_ctx, pos_ctx) + + def unlock(self, file_obj: types.FileArgument) -> None: + import msvcrt + + fd, io_obj_ctx, pos_ctx = _prepare_windows_file(file_obj) + took_fallback_path = False + + try: + msvcrt.locking( # type: ignore[attr-defined] + fd, + msvcrt.LK_UNLCK, # type: ignore[attr-defined] + self._msvcrt_lock_length, + ) + except OSError as exc: + if exc.errno == 13: # EACCES (Permission denied) + took_fallback_path = True + # Restore position before calling win32_locker, + # as it will re-prepare. + _restore_windows_file_pos(io_obj_ctx, pos_ctx) + try: + self._win32_locker.unlock( + file_obj + ) # win32_locker handles its own seeking + except exceptions.LockException as win32_exc: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + f'msvcrt unlock failed ({exc.strerror}), and ' + f'win32 fallback failed ({win32_exc.strerror})', + fh=file_obj, + ) from win32_exc + except Exception as final_exc: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + f'msvcrt unlock failed ({exc.strerror}), and ' + f'win32 fallback failed with unexpected error: ' + f'{final_exc!s}', + fh=file_obj, + ) from final_exc + else: + raise exceptions.LockException( + exceptions.LockException.LOCK_FAILED, + exc.strerror, + fh=file_obj, + ) from exc + finally: + if not took_fallback_path: + _restore_windows_file_pos(io_obj_ctx, pos_ctx) + + _locker_instances: dict[type[BaseLocker], BaseLocker] = dict() + + LOCKER = MsvcrtLocker # type: ignore[reportConstantRedefinition] + + def lock(file: types.FileArgument, flags: LockFlags) -> None: + if isinstance(LOCKER, BaseLocker): + # If LOCKER is a BaseLocker instance, use its lock method + locker: Callable[[types.FileArgument, LockFlags], None] = ( + LOCKER.lock + ) + elif isinstance(LOCKER, tuple): + locker = LOCKER[0] # type: ignore[reportUnknownVariableType] + elif issubclass(LOCKER, BaseLocker): # type: ignore[unreachable,arg-type] # pyright: ignore [reportUnnecessaryIsInstance] + locker_instance = _locker_instances.get(LOCKER) # type: ignore[arg-type] + if locker_instance is None: + # Create an instance of the locker class if not already done + _locker_instances[LOCKER] = locker_instance = LOCKER() # type: ignore[ignore,index,call-arg] + + locker = locker_instance.lock + else: + raise TypeError( + f'LOCKER must be a BaseLocker instance, a tuple of lock and ' + f'unlock functions, or a subclass of BaseLocker, ' + f'got {type(LOCKER)}.' + ) + + locker(file, flags) + + def unlock(file: types.FileArgument) -> None: + if isinstance(LOCKER, BaseLocker): + # If LOCKER is a BaseLocker instance, use its lock method + unlocker: Callable[[types.FileArgument], None] = LOCKER.unlock + elif isinstance(LOCKER, tuple): + unlocker = LOCKER[1] # type: ignore[reportUnknownVariableType] + + elif issubclass(LOCKER, BaseLocker): # type: ignore[unreachable,arg-type] # pyright: ignore [reportUnnecessaryIsInstance] + locker_instance = _locker_instances.get(LOCKER) # type: ignore[arg-type] + if locker_instance is None: + # Create an instance of the locker class if not already done + _locker_instances[LOCKER] = locker_instance = LOCKER() # type: ignore[ignore,index,call-arg] + + unlocker = locker_instance.unlock + else: + raise TypeError( + f'LOCKER must be a BaseLocker instance, a tuple of lock and ' + f'unlock functions, or a subclass of BaseLocker, ' + f'got {type(LOCKER)}.' + ) + + unlocker(file) + +else: # pragma: not-nt + import errno + import fcntl + + # PosixLocker methods accept FileArgument | HasFileno + PosixFileArgument = Union[types.FileArgument, types.HasFileno] + + class PosixLocker(BaseLocker): + """Locker implementation using the `LOCKER` constant""" + + _locker: Optional[ + Callable[[Union[int, types.HasFileno], int], Any] + ] = None + + @property + def locker(self) -> Callable[[Union[int, types.HasFileno], int], Any]: + if self._locker is None: + # On POSIX systems ``LOCKER`` is a callable (fcntl.flock) but + # mypy also sees the Windows-only tuple assignment. Explicitly + # cast so mypy knows we are returning the callable variant + # here. + return cast( + Callable[[Union[int, types.HasFileno], int], Any], LOCKER + ) # pyright: ignore[reportUnnecessaryCast] + + # mypy does not realise ``self._locker`` is non-None after the + # check + assert self._locker is not None + return self._locker + + def _get_fd(self, file_obj: PosixFileArgument) -> int: + if isinstance(file_obj, int): + return file_obj + # Check for fileno() method; covers typing.IO and HasFileno + elif hasattr(file_obj, 'fileno') and callable(file_obj.fileno): + return file_obj.fileno() + else: + # Should not be reached if PosixFileArgument is correct. + # isinstance(file_obj, io.IOBase) could be an + # alternative check + # but hasattr is more general for HasFileno. + raise TypeError( + "Argument 'file_obj' must be an int, an IO object " + 'with fileno(), or implement HasFileno.' + ) + + def lock(self, file_obj: PosixFileArgument, flags: LockFlags) -> None: + if (flags & LockFlags.NON_BLOCKING) and not flags & ( + LockFlags.SHARED | LockFlags.EXCLUSIVE + ): + raise RuntimeError( + 'When locking in non-blocking mode on POSIX, ' + 'the SHARED or EXCLUSIVE flag must be specified as well.' + ) + + fd = self._get_fd(file_obj) + try: + self.locker(fd, flags) + except OSError as exc_value: + if exc_value.errno in (errno.EACCES, errno.EAGAIN): + raise exceptions.AlreadyLocked( + exc_value, + strerror=str(exc_value), + fh=file_obj, # Pass original file_obj + ) from exc_value + else: + raise exceptions.LockException( + exc_value, + strerror=str(exc_value), + fh=file_obj, # Pass original file_obj + ) from exc_value + except EOFError as exc_value: # NFS specific + raise exceptions.LockException( + exc_value, + strerror=str(exc_value), + fh=file_obj, # Pass original file_obj + ) from exc_value + + def unlock(self, file_obj: PosixFileArgument) -> None: + fd = self._get_fd(file_obj) + self.locker(fd, LockFlags.UNBLOCK) + + class FlockLocker(PosixLocker): + """FlockLocker is a PosixLocker implementation using fcntl.flock.""" + + LOCKER = fcntl.flock # type: ignore[attr-defined] + + class LockfLocker(PosixLocker): + """LockfLocker is a PosixLocker implementation using fcntl.lockf.""" + + LOCKER = fcntl.lockf # type: ignore[attr-defined] + + # LOCKER constant for POSIX is fcntl.flock for backward compatibility. + # Type matches: Callable[[Union[int, HasFileno], int], Any] + LOCKER = fcntl.flock # type: ignore[attr-defined,reportConstantRedefinition] + + _posix_locker_instance = PosixLocker() + + # Public API for POSIX uses the PosixLocker instance + def lock(file: types.FileArgument, flags: LockFlags) -> None: + _posix_locker_instance.lock(file, flags) + + def unlock(file: types.FileArgument) -> None: + _posix_locker_instance.unlock(file) diff --git a/contrib/python/portalocker/py3/portalocker/py.typed b/contrib/python/portalocker/py3/portalocker/py.typed new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/py.typed diff --git a/contrib/python/portalocker/py3/portalocker/redis.py b/contrib/python/portalocker/py3/portalocker/redis.py new file mode 100644 index 00000000000..f2e11e6b10b --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/redis.py @@ -0,0 +1,270 @@ +# pyright: reportUnknownMemberType=false +from __future__ import annotations + +import _thread +import json +import logging +import random +import time +import typing + +import redis + +from . import exceptions, utils + +logger = logging.getLogger(__name__) + +DEFAULT_UNAVAILABLE_TIMEOUT = 1 +DEFAULT_THREAD_SLEEP_TIME = 0.1 + + +class PubSubWorkerThread(redis.client.PubSubWorkerThread): + def run(self) -> None: + try: + super().run() + except Exception: # pragma: no cover + _thread.interrupt_main() + raise + + +class RedisLock(utils.LockBase): + """ + An extremely reliable Redis lock based on pubsub with a keep-alive thread + + As opposed to most Redis locking systems based on key/value pairs, + this locking method is based on the pubsub system. The big advantage is + that if the connection gets killed due to network issues, crashing + processes or otherwise, it will still immediately unlock instead of + waiting for a lock timeout. + + To make sure both sides of the lock know about the connection state it is + recommended to set the `health_check_interval` when creating the redis + connection.. + + Args: + channel: the redis channel to use as locking key. + connection: an optional redis connection if you already have one + or if you need to specify the redis connection + timeout: timeout when trying to acquire a lock + check_interval: check interval while waiting + fail_when_locked: after the initial lock failed, return an error + or lock the file. This does not wait for the timeout. + thread_sleep_time: sleep time between fetching messages from redis to + prevent a busy/wait loop. In the case of lock conflicts this + increases the time it takes to resolve the conflict. This should + be smaller than the `check_interval` to be useful. + unavailable_timeout: If the conflicting lock is properly connected + this should never exceed twice your redis latency. Note that this + will increase the wait time possibly beyond your `timeout` and is + always executed if a conflict arises. + redis_kwargs: The redis connection arguments if no connection is + given. The `DEFAULT_REDIS_KWARGS` are used as default, if you want + to override these you need to explicitly specify a value (e.g. + `health_check_interval=0`) + + """ + + redis_kwargs: dict[str, typing.Any] + thread: PubSubWorkerThread | None + channel: str + timeout: float + connection: redis.client.Redis[str] | None + pubsub: redis.client.PubSub | None = None + close_connection: bool + + DEFAULT_REDIS_KWARGS: typing.ClassVar[dict[str, typing.Any]] = dict( + health_check_interval=10, + decode_responses=True, + ) + + def __init__( + self, + channel: str, + connection: redis.client.Redis[str] | None = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = False, + thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, + unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, + redis_kwargs: dict[str, typing.Any] | None = None, + ) -> None: + # We don't want to close connections given as an argument + self.close_connection = not connection + + self.thread = None + self.channel = channel + self.connection = connection + self.thread_sleep_time = thread_sleep_time + self.unavailable_timeout = unavailable_timeout + self.redis_kwargs = redis_kwargs or dict() + + for key, value in self.DEFAULT_REDIS_KWARGS.items(): + self.redis_kwargs.setdefault(key, value) + + super().__init__( + timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, + ) + + def get_connection(self) -> redis.client.Redis[str]: + if not self.connection: + self.connection = redis.client.Redis(**self.redis_kwargs) + + return self.connection + + def channel_handler(self, message: dict[str, str]) -> None: + if message.get('type') != 'message': # pragma: no cover + return + + raw_data = message.get('data') + if not raw_data: + return + + try: + data = json.loads(raw_data) + except TypeError: # pragma: no cover + logger.debug('TypeError while parsing: %r', message) + return + + assert self.connection is not None + self.connection.publish(data['response_channel'], str(time.time())) + + @property + def client_name(self) -> str: + return f'{self.channel}-lock' + + def _timeout_generator( + self, timeout: float | None, check_interval: float | None + ) -> typing.Iterator[int]: + if timeout is None: + timeout = 0.0 + if check_interval is None: + check_interval = self.thread_sleep_time + deadline = time.monotonic() + timeout + first = True + while first or time.monotonic() < deadline: + first = False + effective_interval = ( + check_interval + if check_interval > 0 + else self.thread_sleep_time + ) + sleep_time = effective_interval * (0.5 + random.random()) + time.sleep(sleep_time) + yield 0 + + def acquire( # type: ignore[override] + self, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> RedisLock: + timeout = utils.coalesce(timeout, self.timeout, 0.0) + check_interval = utils.coalesce( + check_interval, + self.check_interval, + 0.0, + ) + fail_when_locked = utils.coalesce( + fail_when_locked, + self.fail_when_locked, + ) + + assert not self.pubsub, 'This lock is already active' + connection = self.get_connection() + + timeout_generator = self._timeout_generator(timeout, check_interval) + for _ in timeout_generator: # pragma: no branch + subscribers = connection.pubsub_numsub(self.channel)[0][1] + + if subscribers: + logger.debug( + 'Found %d lock subscribers for %s', + subscribers, + self.channel, + ) + + if self.check_or_kill_lock( + connection, + self.unavailable_timeout, + ): # pragma: no branch + continue + else: # pragma: no cover + subscribers = 0 + + # Note: this should not be changed to an elif because the if + # above can still end up here + if not subscribers: + connection.client_setname(self.client_name) + self.pubsub = connection.pubsub() + self.pubsub.subscribe(**{self.channel: self.channel_handler}) + self.thread = PubSubWorkerThread( + self.pubsub, + sleep_time=self.thread_sleep_time, + ) + self.thread.start() + time.sleep(0.01) + subscribers = connection.pubsub_numsub(self.channel)[0][1] + if subscribers == 1: # pragma: no branch + return self + else: # pragma: no cover + # Race condition, let's try again + self.release() + + if fail_when_locked: # pragma: no cover + raise exceptions.AlreadyLocked() + + raise exceptions.AlreadyLocked() + + def check_or_kill_lock( + self, + connection: redis.client.Redis[str], + timeout: float, + ) -> bool | None: + # Random channel name to get messages back from the lock + response_channel = f'{self.channel}-{random.random()}' + + pubsub = connection.pubsub() + pubsub.subscribe(response_channel) + connection.publish( + self.channel, + json.dumps( + dict( + response_channel=response_channel, + message='ping', + ), + ), + ) + + check_interval = min(self.thread_sleep_time, timeout / 10) + for _ in self._timeout_generator( + timeout, + check_interval, + ): # pragma: no branch + if pubsub.get_message(timeout=check_interval): + pubsub.close() + return True + + for client_ in connection.client_list('pubsub'): # pragma: no cover + if client_.get('name') == self.client_name: + logger.warning('Killing unavailable redis client: %r', client_) + connection.client_kill_filter( # pyright: ignore + client_.get('id'), + ) + return None + + def release(self) -> None: + if self.thread: # pragma: no branch + self.thread.stop() + self.thread.join() + self.thread = None + time.sleep(0.01) + + if self.pubsub: # pragma: no branch + self.pubsub.unsubscribe(self.channel) + self.pubsub.close() + self.pubsub = None + + def __del__(self) -> None: + self.release() diff --git a/contrib/python/portalocker/py3/portalocker/types.py b/contrib/python/portalocker/py3/portalocker/types.py new file mode 100644 index 00000000000..c08d426e0c2 --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/types.py @@ -0,0 +1,75 @@ +# noqa: A005 +from __future__ import annotations + +import io +import pathlib +import typing +from typing import Union + +# spellchecker: off +# fmt: off +Mode = typing.Literal[ + # Text modes + # Read text + 'r', 'rt', 'tr', + # Write text + 'w', 'wt', 'tw', + # Append text + 'a', 'at', 'ta', + # Exclusive creation text + 'x', 'xt', 'tx', + # Read and write text + 'r+', '+r', 'rt+', 'r+t', '+rt', 'tr+', 't+r', '+tr', + # Write and read text + 'w+', '+w', 'wt+', 'w+t', '+wt', 'tw+', 't+w', '+tw', + # Append and read text + 'a+', '+a', 'at+', 'a+t', '+at', 'ta+', 't+a', '+ta', + # Exclusive creation and read text + 'x+', '+x', 'xt+', 'x+t', '+xt', 'tx+', 't+x', '+tx', + # Universal newline support + 'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr', + + # Binary modes + # Read binary + 'rb', 'br', + # Write binary + 'wb', 'bw', + # Append binary + 'ab', 'ba', + # Exclusive creation binary + 'xb', 'bx', + # Read and write binary + 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br', + # Write and read binary + 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw', + # Append and read binary + 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba', + # Exclusive creation and read binary + 'xb+', 'x+b', '+xb', 'bx+', 'b+x', '+bx', + # Universal newline support in binary mode + 'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr', +] +# spellchecker: on +Filename = Union[str, pathlib.Path] +IO = Union[ # type: ignore[name-defined] + typing.IO[str], + typing.IO[bytes], +] + + +class FileOpenKwargs(typing.TypedDict): + buffering: int | None + encoding: str | None + errors: str | None + newline: str | None + closefd: bool | None + opener: typing.Callable[[str, int], int] | None + + +# Protocol for objects with a fileno() method. +# Used for type-hinting fcntl.flock. +class HasFileno(typing.Protocol): + def fileno(self) -> int: ... + +# Type alias for file arguments used in lock/unlock functions +FileArgument = Union[typing.IO[typing.Any], io.TextIOWrapper, int, HasFileno] diff --git a/contrib/python/portalocker/py3/portalocker/utils.py b/contrib/python/portalocker/py3/portalocker/utils.py new file mode 100644 index 00000000000..b6dccb1404b --- /dev/null +++ b/contrib/python/portalocker/py3/portalocker/utils.py @@ -0,0 +1,586 @@ +from __future__ import annotations + +import abc +import atexit +import contextlib +import logging +import os +import pathlib +import random +import tempfile +import time +import typing +import warnings + +from . import constants, exceptions, portalocker, types +from .types import Filename, Mode + +logger = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 5 +DEFAULT_CHECK_INTERVAL = 0.25 +DEFAULT_FAIL_WHEN_LOCKED = False +LOCK_METHOD = constants.LockFlags.EXCLUSIVE | constants.LockFlags.NON_BLOCKING + +__all__ = [ + 'Lock', + 'open_atomic', +] + + +def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: + """Simple coalescing function that returns the first value that is not + equal to the `test_value`. Or `None` if no value is valid. Usually this + means that the last given value is the default value. + + Note that the `test_value` is compared using an identity check + (i.e. `value is not test_value`) so changing the `test_value` won't work + for all values. + + >>> coalesce(None, 1) + 1 + >>> coalesce() + + >>> coalesce(0, False, True) + 0 + >>> coalesce(0, False, True, test_value=0) + False + + # This won't work because of the `is not test_value` type testing: + >>> coalesce([], dict(spam='eggs'), test_value=[]) + [] + """ + return next((arg for arg in args if arg is not test_value), None) + + +def open_atomic( + filename: Filename, + binary: bool = True, +) -> typing.Iterator[types.IO]: + """Open a file for atomic writing. Instead of locking this method allows + you to write the entire file and move it to the actual location. Note that + this makes the assumption that a rename is atomic on your platform which + is generally the case but not a guarantee. + + http://docs.python.org/library/os.html#os.rename + + >>> filename = 'test_file.txt' + >>> if os.path.exists(filename): + ... os.remove(filename) + + >>> with open_atomic(filename) as fh: + ... written = fh.write(b'test') + >>> assert os.path.exists(filename) + >>> os.remove(filename) + + >>> import pathlib + >>> path_filename = pathlib.Path('test_file.txt') + + >>> with open_atomic(path_filename) as fh: + ... written = fh.write(b'test') + >>> assert path_filename.exists() + >>> path_filename.unlink() + """ + # `pathlib.Path` cast in case `path` is a `str` + path: pathlib.Path + if isinstance(filename, pathlib.Path): + path = filename + else: + path = pathlib.Path(filename) + + assert not path.exists(), f'{path!r} exists' + + # Create the parent directory if it doesn't exist + path.parent.mkdir(parents=True, exist_ok=True) + + with tempfile.NamedTemporaryFile( + mode=(binary and 'wb') or 'w', + dir=str(path.parent), + delete=False, + ) as temp_fh: + yield temp_fh + temp_fh.flush() + os.fsync(temp_fh.fileno()) + + try: + os.rename(temp_fh.name, path) + finally: + with contextlib.suppress(Exception): + os.remove(temp_fh.name) + + +class LockBase(abc.ABC): # pragma: no cover + #: timeout when trying to acquire a lock + timeout: float + #: check interval while waiting for `timeout` + check_interval: float + #: skip the timeout and immediately fail if the initial lock fails + fail_when_locked: bool + + def __init__( + self, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> None: + self.timeout = coalesce(timeout, DEFAULT_TIMEOUT) + self.check_interval = coalesce(check_interval, DEFAULT_CHECK_INTERVAL) + self.fail_when_locked = coalesce( + fail_when_locked, + DEFAULT_FAIL_WHEN_LOCKED, + ) + + @abc.abstractmethod + def acquire( + self, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> typing.IO[typing.AnyStr]: ... + + def _timeout_generator( + self, + timeout: float | None, + check_interval: float | None, + ) -> typing.Iterator[int]: + f_timeout = coalesce(timeout, self.timeout, 0.0) + f_check_interval = coalesce(check_interval, self.check_interval, 0.0) + + yield 0 + i = 0 + + start_time = time.perf_counter() + while start_time + f_timeout > time.perf_counter(): + i += 1 + yield i + + # Take low lock checks into account to stay within the interval + since_start_time = time.perf_counter() - start_time + time.sleep(max(0.001, (i * f_check_interval) - since_start_time)) + + @abc.abstractmethod + def release(self) -> None: ... + + def __enter__(self) -> typing.IO[typing.AnyStr]: + return self.acquire() + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: typing.Any, # Should be typing.TracebackType + ) -> bool | None: + self.release() + return None + + def __delete__(self, instance: LockBase) -> None: + instance.release() + + +class Lock(LockBase): + """Lock manager with built-in timeout + + Args: + filename: filename + mode: the open mode, 'a' or 'ab' should be used for writing. When mode + contains `w` the file will be truncated to 0 bytes. + timeout: timeout when trying to acquire a lock + check_interval: check interval while waiting + fail_when_locked: after the initial lock failed, return an error + or lock the file. This does not wait for the timeout. + **file_open_kwargs: The kwargs for the `open(...)` call + + fail_when_locked is useful when multiple threads/processes can race + when creating a file. If set to true than the system will wait till + the lock was acquired and then return an AlreadyLocked exception. + + Note that the file is opened first and locked later. So using 'w' as + mode will result in truncate _BEFORE_ the lock is checked. + """ + + fh: types.IO | None + filename: str + mode: str + truncate: bool + timeout: float + check_interval: float + fail_when_locked: bool + flags: constants.LockFlags + file_open_kwargs: dict[str, typing.Any] + + def __init__( + self, + filename: Filename, + mode: Mode = 'a', + timeout: float | None = None, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, + flags: constants.LockFlags = LOCK_METHOD, + **file_open_kwargs: typing.Any, + ) -> None: + if 'w' in mode: + truncate = True + mode = typing.cast(Mode, mode.replace('w', 'a')) + else: + truncate = False + + if timeout is None: + timeout = DEFAULT_TIMEOUT + elif not (flags & constants.LockFlags.NON_BLOCKING): + warnings.warn( + 'timeout has no effect in blocking mode', + stacklevel=1, + ) + + self.fh = None + self.filename = str(filename) + self.mode = mode + self.truncate = truncate + self.flags = flags + self.file_open_kwargs = file_open_kwargs + super().__init__(timeout, check_interval, fail_when_locked) + + def acquire( + self, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> typing.IO[typing.AnyStr]: + """Acquire the locked filehandle""" + + fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) + + if ( + not (self.flags & constants.LockFlags.NON_BLOCKING) + and timeout is not None + ): + warnings.warn( + 'timeout has no effect in blocking mode', + stacklevel=1, + ) + + # If we already have a filehandle, return it + fh = self.fh + if fh: + # Due to type invariance we need to cast the type + return typing.cast(typing.IO[typing.AnyStr], fh) + + # Get a new filehandler + fh = self._get_fh() + + def try_close() -> None: # pragma: no cover + # Silently try to close the handle if possible, ignore all issues + if fh is not None: + with contextlib.suppress(Exception): + fh.close() + + exception = None + # Try till the timeout has passed + for _ in self._timeout_generator(timeout, check_interval): + exception = None + try: + # Try to lock + fh = self._get_lock(fh) + break + except exceptions.LockException as exc: + # Python will automatically remove the variable from memory + # unless you save it in a different location + exception = exc + + # We already tried to the get the lock + # If fail_when_locked is True, stop trying + if fail_when_locked: + try_close() + raise exceptions.AlreadyLocked(exception) from exc + except Exception as exc: + # Something went wrong with the locking mechanism. + # Wrap in a LockException and re-raise: + try_close() + raise exceptions.LockException(exc) from exc + + # Wait a bit + + if exception: + try_close() + # We got a timeout... reraising + raise exception + + # Prepare the filehandle (truncate if needed) + fh = self._prepare_fh(fh) + + self.fh = fh + return typing.cast(typing.IO[typing.AnyStr], fh) + + def __enter__(self) -> typing.IO[typing.AnyStr]: + return self.acquire() + + def release(self) -> None: + """Releases the currently locked file handle""" + if self.fh: + portalocker.unlock(self.fh) + self.fh.close() + self.fh = None + + def _get_fh(self) -> types.IO: + """Get a new filehandle""" + return typing.cast( + types.IO, + open( # noqa: SIM115 + self.filename, + self.mode, + **self.file_open_kwargs, + ), + ) + + def _get_lock(self, fh: types.IO) -> types.IO: + """ + Try to lock the given filehandle + + returns LockException if it fails""" + portalocker.lock(fh, self.flags) + return fh + + def _prepare_fh(self, fh: types.IO) -> types.IO: + """ + Prepare the filehandle for usage + + If truncate is a number, the file will be truncated to that amount of + bytes + """ + if self.truncate: + fh.seek(0) + fh.truncate(0) + + return fh + + +class RLock(Lock): + """ + A reentrant lock, functions in a similar way to threading.RLock in that it + can be acquired multiple times. When the corresponding number of release() + calls are made the lock will finally release the underlying file lock. + """ + + def __init__( + self, + filename: Filename, + mode: Mode = 'a', + timeout: float = DEFAULT_TIMEOUT, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = False, + flags: constants.LockFlags = LOCK_METHOD, + ) -> None: + super().__init__( + filename, + mode, + timeout, + check_interval, + fail_when_locked, + flags, + ) + self._acquire_count = 0 + + def acquire( + self, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> typing.IO[typing.AnyStr]: + fh: typing.IO[typing.AnyStr] + if self._acquire_count >= 1: + fh = typing.cast(typing.IO[typing.AnyStr], self.fh) + else: + fh = super().acquire(timeout, check_interval, fail_when_locked) + self._acquire_count += 1 + assert fh is not None + return fh + + def release(self) -> None: + if self._acquire_count == 0: + raise exceptions.LockException( + 'Cannot release more times than acquired', + ) + + if self._acquire_count == 1: + super().release() + self._acquire_count -= 1 + + +class TemporaryFileLock(Lock): + def __init__( + self, + filename: str = '.lock', + timeout: float = DEFAULT_TIMEOUT, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = True, + flags: constants.LockFlags = LOCK_METHOD, + ) -> None: + super().__init__( + filename=filename, + mode='w', + timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, + flags=flags, + ) + atexit.register(self.release) + + def release(self) -> None: + Lock.release(self) + if os.path.isfile(self.filename): # pragma: no branch + os.unlink(self.filename) + + +class BoundedSemaphore(LockBase): + """ + Bounded semaphore to prevent too many parallel processes from running + + This method is deprecated because multiple processes that are completely + unrelated could end up using the same semaphore. To prevent this, + use `NamedBoundedSemaphore` instead. The + `NamedBoundedSemaphore` is a drop-in replacement for this class. + + >>> semaphore = BoundedSemaphore(2, directory='') + >>> str(semaphore.get_filenames()[0]) + 'bounded_semaphore.00.lock' + >>> str(sorted(semaphore.get_random_filenames())[1]) + 'bounded_semaphore.01.lock' + """ + + lock: Lock | None + + def __init__( + self, + maximum: int, + name: str = 'bounded_semaphore', + filename_pattern: str = '{name}.{number:02d}.lock', + directory: str = tempfile.gettempdir(), + timeout: float | None = DEFAULT_TIMEOUT, + check_interval: float | None = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool | None = True, + ) -> None: + self.maximum = maximum + self.name = name + self.filename_pattern = filename_pattern + self.directory = directory + self.lock: Lock | None = None + super().__init__( + timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, + ) + + if not name or name == 'bounded_semaphore': + warnings.warn( + '`BoundedSemaphore` without an explicit `name` ' + 'argument is deprecated, use NamedBoundedSemaphore', + DeprecationWarning, + stacklevel=1, + ) + + def get_filenames(self) -> typing.Sequence[pathlib.Path]: + return [self.get_filename(n) for n in range(self.maximum)] + + def get_random_filenames(self) -> typing.Sequence[pathlib.Path]: + filenames = list(self.get_filenames()) + random.shuffle(filenames) + return filenames + + def get_filename(self, number: int) -> pathlib.Path: + return pathlib.Path(self.directory) / self.filename_pattern.format( + name=self.name, + number=number, + ) + + def acquire( # type: ignore[override] + self, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> Lock | None: + assert not self.lock, 'Already locked' + + filenames = self.get_filenames() + + for n in self._timeout_generator(timeout, check_interval): # pragma: + logger.debug('trying lock (attempt %d) %r', n, filenames) + # no branch + if self.try_lock(filenames): # pragma: no branch + return self.lock # pragma: no cover + + if fail_when_locked := coalesce( + fail_when_locked, + self.fail_when_locked, + ): + raise exceptions.AlreadyLocked() + + return None + + def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: + filename: Filename + for filename in filenames: + logger.debug('trying lock for %r', filename) + self.lock = Lock(filename, fail_when_locked=True) + try: + self.lock.acquire() + except exceptions.AlreadyLocked: + self.lock = None + else: + logger.debug('locked %r', filename) + return True + + return False + + def release(self) -> None: # pragma: no cover + if self.lock is not None: + self.lock.release() + self.lock = None + + +class NamedBoundedSemaphore(BoundedSemaphore): + """ + Bounded semaphore to prevent too many parallel processes from running + + It's also possible to specify a timeout when acquiring the lock to wait + for a resource to become available. This is very similar to + `threading.BoundedSemaphore` but works across multiple processes and across + multiple operating systems. + + Because this works across multiple processes it's important to give the + semaphore a name. This name is used to create the lock files. If you + don't specify a name, a random name will be generated. This means that + you can't use the same semaphore in multiple processes unless you pass the + semaphore object to the other processes. + + >>> semaphore = NamedBoundedSemaphore(2, name='test') + >>> str(semaphore.get_filenames()[0]) + '...test.00.lock' + + >>> semaphore = NamedBoundedSemaphore(2) + >>> 'bounded_semaphore' in str(semaphore.get_filenames()[0]) + True + + """ + + def __init__( + self, + maximum: int, + name: str | None = None, + filename_pattern: str = '{name}.{number:02d}.lock', + directory: str = tempfile.gettempdir(), + timeout: float | None = DEFAULT_TIMEOUT, + check_interval: float | None = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool | None = True, + ) -> None: + if name is None: + name = f'bounded_semaphore.{random.randint(0, 1000000):d}' + super().__init__( + maximum, + name, + filename_pattern, + directory, + timeout, + check_interval, + fail_when_locked, + ) diff --git a/contrib/python/portalocker/py3/tests/__init__.py b/contrib/python/portalocker/py3/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/portalocker/py3/tests/__init__.py diff --git a/contrib/python/portalocker/py3/tests/conftest.py b/contrib/python/portalocker/py3/tests/conftest.py new file mode 100644 index 00000000000..cf59e2b32f4 --- /dev/null +++ b/contrib/python/portalocker/py3/tests/conftest.py @@ -0,0 +1,23 @@ +import logging +import pytest +import random +import multiprocessing + +logger = logging.getLogger(__name__) + + +def tmpfile(tmp_path): + filename = tmp_path / str(random.random()) + yield str(filename) + try: + filename.unlink(missing_ok=True) + except PermissionError: + pass + + +def pytest_sessionstart(session): + # Force spawning the process so we don't accidently inherit locks. + # I'm not a 100% certain this will work correctly unfortunately... there + # is some potential for breaking tests + multiprocessing.set_start_method('spawn') diff --git a/contrib/python/portalocker/py3/tests/temporary_file_lock.py b/contrib/python/portalocker/py3/tests/temporary_file_lock.py new file mode 100644 index 00000000000..b250bad6510 --- /dev/null +++ b/contrib/python/portalocker/py3/tests/temporary_file_lock.py @@ -0,0 +1,14 @@ +import os +import portalocker + + +def test_temporary_file_lock(tmpfile): + with portalocker.TemporaryFileLock(tmpfile): + pass + + assert not os.path.isfile(tmpfile) + + lock = portalocker.TemporaryFileLock(tmpfile) + lock.acquire() + del lock + diff --git a/contrib/python/portalocker/py3/tests/test_combined.py b/contrib/python/portalocker/py3/tests/test_combined.py new file mode 100644 index 00000000000..594de74b9ba --- /dev/null +++ b/contrib/python/portalocker/py3/tests/test_combined.py @@ -0,0 +1,15 @@ +import sys + + +def test_combined(tmpdir): + from distutils import dist + import setup + + output_file = tmpdir.join('combined.py') + combine = setup.Combine(dist.Distribution()) + combine.output_file = str(output_file) + combine.run() + sys.path.append(output_file.dirname) + import combined + assert combined + diff --git a/contrib/python/portalocker/py3/tests/test_redis.py b/contrib/python/portalocker/py3/tests/test_redis.py new file mode 100644 index 00000000000..694c9af17ca --- /dev/null +++ b/contrib/python/portalocker/py3/tests/test_redis.py @@ -0,0 +1,90 @@ +import _thread +import logging +import random +import time + +import pytest +from redis import client +from redis import exceptions + +import portalocker +from portalocker import redis +from portalocker import utils + +logger = logging.getLogger(__name__) + +try: + client.Redis().ping() +except (exceptions.ConnectionError, ConnectionRefusedError): + pytest.skip('Unable to connect to redis', allow_module_level=True) + + [email protected](autouse=True) +def set_redis_timeouts(monkeypatch): + monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) + monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) + monkeypatch.setattr(redis, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) + monkeypatch.setattr(redis, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) + monkeypatch.setattr(_thread, 'interrupt_main', lambda: None) + + +def test_redis_lock(): + channel = str(random.random()) + + lock_a = redis.RedisLock(channel) + lock_a.acquire(fail_when_locked=True) + time.sleep(0.01) + + lock_b = redis.RedisLock(channel) + try: + with pytest.raises(portalocker.AlreadyLocked): + lock_b.acquire(fail_when_locked=True) + finally: + lock_a.release() + lock_a.connection.close() + + [email protected]('timeout', [None, 0, 0.001]) [email protected]('check_interval', [None, 0, 0.0005]) +def test_redis_lock_timeout(timeout, check_interval): + connection = client.Redis() + channel = str(random.random()) + lock_a = redis.RedisLock(channel) + lock_a.acquire(timeout=timeout, check_interval=check_interval) + + lock_b = redis.RedisLock(channel, connection=connection) + with pytest.raises(portalocker.AlreadyLocked): + try: + lock_b.acquire(timeout=timeout, check_interval=check_interval) + finally: + lock_a.release() + lock_a.connection.close() + + +def test_redis_lock_context(): + channel = str(random.random()) + + lock_a = redis.RedisLock(channel, fail_when_locked=True) + with lock_a: + time.sleep(0.01) + lock_b = redis.RedisLock(channel, fail_when_locked=True) + with pytest.raises(portalocker.AlreadyLocked): + with lock_b: + pass + + +def test_redis_relock(): + channel = str(random.random()) + + lock_a = redis.RedisLock(channel, fail_when_locked=True) + with lock_a: + time.sleep(0.01) + with pytest.raises(AssertionError): + lock_a.acquire() + time.sleep(0.01) + + lock_a.release() + + +if __name__ == '__main__': + test_redis_lock() diff --git a/contrib/python/portalocker/py3/tests/test_semaphore.py b/contrib/python/portalocker/py3/tests/test_semaphore.py new file mode 100644 index 00000000000..b0c57aa23c7 --- /dev/null +++ b/contrib/python/portalocker/py3/tests/test_semaphore.py @@ -0,0 +1,22 @@ +import random +import pytest +import portalocker +from portalocker import utils + + [email protected]('timeout', [None, 0, 0.001]) [email protected]('check_interval', [None, 0, 0.0005]) +def test_bounded_semaphore(timeout, check_interval, monkeypatch): + n = 2 + name = random.random() + monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) + monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) + + semaphore_a = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) + semaphore_b = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) + semaphore_c = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) + + semaphore_a.acquire(timeout=timeout) + semaphore_b.acquire() + with pytest.raises(portalocker.AlreadyLocked): + semaphore_c.acquire(check_interval=check_interval, timeout=timeout) diff --git a/contrib/python/portalocker/py3/tests/tests.py b/contrib/python/portalocker/py3/tests/tests.py new file mode 100644 index 00000000000..7a00405966e --- /dev/null +++ b/contrib/python/portalocker/py3/tests/tests.py @@ -0,0 +1,357 @@ +from __future__ import print_function +from __future__ import with_statement + +import os +import dataclasses +import multiprocessing +import time +import typing + +import pytest +import portalocker +from portalocker import utils +from portalocker import LockFlags + + +def test_exceptions(tmpfile): + # Open the file 2 times + a = open(tmpfile, 'a') + b = open(tmpfile, 'a') + + # Lock exclusive non-blocking + lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB + + # First lock file a + portalocker.lock(a, lock_flags) + + # Now see if we can lock file b + with pytest.raises(portalocker.LockException): + portalocker.lock(b, lock_flags) + + # Cleanup + a.close() + b.close() + + +def test_utils_base(): + class Test(utils.LockBase): + pass + + +def test_with_timeout(tmpfile): + # Open the file 2 times + with pytest.raises(portalocker.AlreadyLocked): + with portalocker.Lock(tmpfile, timeout=0.1) as fh: + print('writing some stuff to my cache...', file=fh) + with portalocker.Lock( + tmpfile, timeout=0.1, mode='wb', + fail_when_locked=True + ): + pass + print('writing more stuff to my cache...', file=fh) + + +def test_without_timeout(tmpfile): + # Open the file 2 times + with pytest.raises(portalocker.LockException): + with portalocker.Lock(tmpfile, timeout=None) as fh: + print('writing some stuff to my cache...', file=fh) + with portalocker.Lock(tmpfile, timeout=None, mode='w'): + pass + print('writing more stuff to my cache...', file=fh) + + +def test_without_fail(tmpfile): + # Open the file 2 times + with pytest.raises(portalocker.LockException): + with portalocker.Lock(tmpfile, timeout=0.1) as fh: + print('writing some stuff to my cache...', file=fh) + lock = portalocker.Lock(tmpfile, timeout=0.1) + lock.acquire(check_interval=0.05, fail_when_locked=False) + + +def test_simple(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + fh = open(tmpfile, 'r+') + portalocker.lock(fh, portalocker.LOCK_EX) + + fh.seek(13) + fh.write('foo') + + # Make sure we didn't overwrite the original text + fh.seek(0) + assert fh.read(13) == 'spam and eggs' + + portalocker.unlock(fh) + fh.close() + + +def test_truncate(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + with portalocker.Lock(tmpfile, mode='a+') as fh: + # Make sure we didn't overwrite the original text + fh.seek(0) + assert fh.read(13) == 'spam and eggs' + + with portalocker.Lock(tmpfile, mode='w+') as fh: + # Make sure we truncated the file + assert fh.read() == '' + + +def test_class(tmpfile): + lock = portalocker.Lock(tmpfile) + lock2 = portalocker.Lock(tmpfile, fail_when_locked=False, timeout=0.01) + + with lock: + lock.acquire() + + with pytest.raises(portalocker.LockException): + with lock2: + pass + + with lock2: + pass + + +def test_acquire_release(tmpfile): + lock = portalocker.Lock(tmpfile) + lock2 = portalocker.Lock(tmpfile, fail_when_locked=False) + + lock.acquire() # acquire lock when nobody is using it + with pytest.raises(portalocker.LockException): + # another party should not be able to acquire the lock + lock2.acquire(timeout=0.01) + + # re-acquire a held lock is a no-op + lock.acquire() + + lock.release() # release the lock + lock.release() # second release does nothing + + +def test_rlock_acquire_release_count(tmpfile): + lock = portalocker.RLock(tmpfile) + # Twice acquire + h = lock.acquire() + assert not h.closed + lock.acquire() + assert not h.closed + + # Two release + lock.release() + assert not h.closed + lock.release() + assert h.closed + + +def test_rlock_acquire_release(tmpfile): + lock = portalocker.RLock(tmpfile) + lock2 = portalocker.RLock(tmpfile, fail_when_locked=False) + + lock.acquire() # acquire lock when nobody is using it + with pytest.raises(portalocker.LockException): + # another party should not be able to acquire the lock + lock2.acquire(timeout=0.01) + + # Now acquire again + lock.acquire() + + lock.release() # release the lock + lock.release() # second release does nothing + + +def test_release_unacquired(tmpfile): + with pytest.raises(portalocker.LockException): + portalocker.RLock(tmpfile).release() + + +def test_exlusive(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + fh = open(tmpfile, 'r') + portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) + + # Make sure we can't read the locked file + with pytest.raises(portalocker.LockException): + with open(tmpfile, 'r') as fh2: + portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) + fh2.read() + + # Make sure we can't write the locked file + with pytest.raises(portalocker.LockException): + with open(tmpfile, 'w+') as fh2: + portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) + fh2.write('surprise and fear') + + # Make sure we can explicitly unlock the file + portalocker.unlock(fh) + fh.close() + + +def test_shared(tmpfile): + with open(tmpfile, 'w') as fh: + fh.write('spam and eggs') + + f = open(tmpfile, 'r') + portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB) + + # Make sure we can read the locked file + with open(tmpfile, 'r') as fh2: + portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB) + assert fh2.read() == 'spam and eggs' + + # Make sure we can't write the locked file + with pytest.raises(portalocker.LockException): + with open(tmpfile, 'w+') as fh2: + portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) + fh2.write('surprise and fear') + + # Make sure we can explicitly unlock the file + portalocker.unlock(f) + f.close() + + +def test_blocking_timeout(tmpfile): + flags = LockFlags.SHARED + + with pytest.warns(UserWarning): + with portalocker.Lock(tmpfile, timeout=5, flags=flags): + pass + + lock = portalocker.Lock(tmpfile, flags=flags) + with pytest.warns(UserWarning): + lock.acquire(timeout=5) + + [email protected](os.name == 'nt', + reason='Windows uses an entirely different lockmechanism') +def test_nonblocking(tmpfile): + with open(tmpfile, 'w') as fh: + with pytest.raises(RuntimeError): + portalocker.lock(fh, LockFlags.NON_BLOCKING) + + +def shared_lock(filename, **kwargs): + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=False, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, + ): + time.sleep(0.2) + return True + + +def shared_lock_fail(filename, **kwargs): + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=True, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, + ): + time.sleep(0.2) + return True + + +def exclusive_lock(filename, **kwargs): + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=False, + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, + ): + time.sleep(0.2) + return True + + [email protected](order=True) +class LockResult: + exception_class: typing.Union[typing.Type[Exception], None] = None + exception_message: typing.Union[str, None] = None + exception_repr: typing.Union[str, None] = None + + +def lock( + filename: str, + fail_when_locked: bool, + flags: LockFlags +) -> LockResult: + # Returns a case of True, False or FileNotFound + # https://thedailywtf.com/articles/what_is_truth_0x3f_ + # But seriously, the exception properties cannot be safely pickled so we + # only return string representations of the exception properties + try: + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=fail_when_locked, + flags=flags, + ): + time.sleep(0.2) + return LockResult() + + except Exception as exception: + # The exceptions cannot be pickled so we cannot return them through + # multiprocessing + return LockResult( + type(exception), + str(exception), + repr(exception), + ) + + [email protected]('fail_when_locked', [True, False]) +def test_shared_processes(tmpfile, fail_when_locked): + flags = LockFlags.SHARED | LockFlags.NON_BLOCKING + + with multiprocessing.Pool(processes=2) as pool: + args = tmpfile, fail_when_locked, flags + results = pool.starmap_async(lock, 2 * [args]) + + for result in results.get(timeout=3): + assert result == LockResult() + + [email protected]('fail_when_locked', [True, False]) +def test_exclusive_processes(tmpfile, fail_when_locked): + flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING + + with multiprocessing.Pool(processes=2) as pool: + # filename, fail_when_locked, flags + args = tmpfile, fail_when_locked, flags + a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=3) + + assert not a.exception_class or not b.exception_class + assert issubclass( + a.exception_class or b.exception_class, + portalocker.LockException + ) + + + os.name == 'nt', + reason='Locking on Windows requires a file object', +) +def test_lock_fileno(tmpfile): + # Open the file 2 times + a = open(tmpfile, 'a') + b = open(tmpfile, 'a') + + # Lock exclusive non-blocking + flags = LockFlags.SHARED | LockFlags.NON_BLOCKING + + # First lock file a + portalocker.lock(a, flags) + + # Now see if we can lock using fileno() + portalocker.lock(b.fileno(), flags) + + # Cleanup + a.close() + b.close() + diff --git a/contrib/python/portalocker/py3/tests/ya.make b/contrib/python/portalocker/py3/tests/ya.make new file mode 100644 index 00000000000..342f72a5391 --- /dev/null +++ b/contrib/python/portalocker/py3/tests/ya.make @@ -0,0 +1,23 @@ +PY3TEST() + +SUBSCRIBER(g:python-contrib) + +NO_LINT() + +PEERDIR( + contrib/python/portalocker + contrib/python/redis +) + +TEST_SRCS( + __init__.py + conftest.py + temporary_file_lock.py + # Tests intallation. + # test_combined.py + test_redis.py + test_semaphore.py + tests.py +) + +END() diff --git a/contrib/python/portalocker/py3/ya.make b/contrib/python/portalocker/py3/ya.make new file mode 100644 index 00000000000..0929f09336f --- /dev/null +++ b/contrib/python/portalocker/py3/ya.make @@ -0,0 +1,39 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(3.2.0) + +LICENSE(BSD-3-Clause) + +NO_LINT() + +NO_CHECK_IMPORTS( + portalocker.redis +) + +PY_SRCS( + TOP_LEVEL + portalocker/__about__.py + portalocker/__init__.py + portalocker/__main__.py + portalocker/constants.py + portalocker/exceptions.py + portalocker/portalocker.py + portalocker/redis.py + portalocker/types.py + portalocker/utils.py +) + +RESOURCE_FILES( + PREFIX contrib/python/portalocker/py3/ + .dist-info/METADATA + .dist-info/top_level.txt + portalocker/py.typed +) + +END() + +RECURSE( + tests +) diff --git a/contrib/python/portalocker/ya.make b/contrib/python/portalocker/ya.make new file mode 100644 index 00000000000..119deebfda1 --- /dev/null +++ b/contrib/python/portalocker/ya.make @@ -0,0 +1,20 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +VERSION(Service-proxy-version) + +IF (PYTHON2) + PEERDIR(contrib/python/portalocker/py2) +ELSE() + PEERDIR(contrib/python/portalocker/py3) +ENDIF() + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +) diff --git a/contrib/python/python-libarchive/py2/libarchive/__init__.py b/contrib/python/python-libarchive/py2/libarchive/__init__.py new file mode 100644 index 00000000000..0c0c63359a3 --- /dev/null +++ b/contrib/python/python-libarchive/py2/libarchive/__init__.py @@ -0,0 +1,800 @@ +# Copyright (c) 2011, SmartFile <[email protected]> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the organization nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import stat +import sys +import math +import time +import logging +import warnings + +import contextlib2 + +from libarchive import _libarchive +import six + +logger = logging.getLogger(__name__) + +# Suggested block size for libarchive. Libarchive may adjust it. +BLOCK_SIZE = 10240 + +MTIME_FORMAT = '' + +# Default encoding scheme. +ENCODING = 'utf-8' + +if six.PY2: + def encode(value, encoding): + if type(value) == str: + value = value.decode(encoding, errors='ignore') + return value.encode(encoding) +else: + def encode(value, encoding): + return value.encode(encoding) + + +# Functions to initialize read/write for various libarchive supported formats and filters. +FORMATS = { + None: (_libarchive.archive_read_support_format_all, None), + 'tar': (_libarchive.archive_read_support_format_tar, _libarchive.archive_write_set_format_ustar), + 'pax': (_libarchive.archive_read_support_format_tar, _libarchive.archive_write_set_format_pax), + 'gnu': (_libarchive.archive_read_support_format_gnutar, _libarchive.archive_write_set_format_gnutar), + 'zip': (_libarchive.archive_read_support_format_zip, _libarchive.archive_write_set_format_zip), + 'rar': (_libarchive.archive_read_support_format_rar, None), + '7zip': (_libarchive.archive_read_support_format_7zip, None), + 'ar': (_libarchive.archive_read_support_format_ar, None), + 'cab': (_libarchive.archive_read_support_format_cab, None), + 'cpio': (_libarchive.archive_read_support_format_cpio, _libarchive.archive_write_set_format_cpio_newc), + 'iso': (_libarchive.archive_read_support_format_iso9660, _libarchive.archive_write_set_format_iso9660), + 'lha': (_libarchive.archive_read_support_format_lha, None), + 'xar': (_libarchive.archive_read_support_format_xar, _libarchive.archive_write_set_format_xar), +} + +FILTERS = { + None: (_libarchive.archive_read_support_filter_all, _libarchive.archive_write_add_filter_none), + 'bzip2': (_libarchive.archive_read_support_filter_bzip2, _libarchive.archive_write_add_filter_bzip2), + 'gzip': (_libarchive.archive_read_support_filter_gzip, _libarchive.archive_write_add_filter_gzip), + 'zstd': (_libarchive.archive_read_support_filter_zstd, _libarchive.archive_write_add_filter_zstd), +} + +# Map file extensions to formats and filters. To support quick detection. +FORMAT_EXTENSIONS = { + '.tar': 'tar', + '.zip': 'zip', + '.rar': 'rar', + '.7z': '7zip', + '.ar': 'ar', + '.cab': 'cab', + '.rpm': 'cpio', + '.cpio': 'cpio', + '.iso': 'iso', + '.lha': 'lha', + '.xar': 'xar', +} +FILTER_EXTENSIONS = { + '.bz2': 'bzip2', + '.gz': 'gzip', + '.zst': 'zstd', +} + + +class EOF(Exception): + '''Raised by ArchiveInfo.from_archive() when unable to read the next + archive header.''' + pass + + +def get_error(archive): + '''Retrieves the last error description for the given archive instance.''' + return _libarchive.archive_error_string(archive) + + +def call_and_check(func, archive, *args): + '''Executes a libarchive function and raises an exception when appropriate.''' + ret = func(*args) + if ret == _libarchive.ARCHIVE_OK: + return + elif ret == _libarchive.ARCHIVE_WARN: + warnings.warn('Warning executing function: %s.' % get_error(archive), RuntimeWarning) + elif ret == _libarchive.ARCHIVE_EOF: + raise EOF() + else: + raise Exception('Fatal error executing function, message is: %s.' % get_error(archive)) + + +def get_func(name, items, index): + item = items.get(name, None) + if item is None: + return None + return item[index] + + +def guess_format(filename): + filename, ext = os.path.splitext(filename) + filter = FILTER_EXTENSIONS.get(ext) + if filter: + filename, ext = os.path.splitext(filename) + format = FORMAT_EXTENSIONS.get(ext) + return format, filter + + +def is_archive_name(filename, formats=None): + '''Quick check to see if the given file has an extension indiciating that it is + an archive. The format parameter can be used to limit what archive format is acceptable. + If omitted, all supported archive formats will be checked. + + This function will return the name of the most likely archive format, None if the file is + unlikely to be an archive.''' + if formats is None: + formats = FORMAT_EXTENSIONS.values() + format, filter = guess_format(filename) + if format in formats: + return format + + +def is_archive(f, formats=(None, ), filters=(None, )): + '''Check to see if the given file is actually an archive. The format parameter + can be used to specify which archive format is acceptable. If ommitted, all supported + archive formats will be checked. It opens the file using libarchive. If no error is + received, the file was successfully detected by the libarchive bidding process. + + This procedure is quite costly, so you should avoid calling it unless you are reasonably + sure that the given file is an archive. In other words, you may wish to filter large + numbers of file names using is_archive_name() before double-checking the positives with + this function. + + This function will return True if the file can be opened as an archive using the given + format(s)/filter(s).''' + with contextlib2.ExitStack() as exit_stack: + if isinstance(f, six.string_types): + f = exit_stack.enter_context(open(f, 'rb')) + a = _libarchive.archive_read_new() + for format in formats: + format = get_func(format, FORMATS, 0) + if format is None: + return False + format(a) + for filter in filters: + filter = get_func(filter, FILTERS, 0) + if filter is None: + return False + filter(a) + try: + try: + call_and_check(_libarchive.archive_read_open_fd, a, a, f.fileno(), BLOCK_SIZE) + return True + except: + return False + finally: + _libarchive.archive_read_close(a) + _libarchive.archive_read_free(a) + + +def get_archive_filter_names(filename): + with open(filename, 'rb') as afile: + a = _libarchive.archive_read_new() + try: + format_func = get_func(None, FORMATS, 0) + format_func(a) + filter_func = get_func(None, FILTERS, 0) + filter_func(a) + if _libarchive.archive_read_open_fd(a, afile.fileno(), BLOCK_SIZE) == _libarchive.ARCHIVE_OK: + try: + nfilter = _libarchive.archive_filter_count(a) + return [_libarchive.archive_filter_name(a, i).decode(ENCODING) for i in range(nfilter)] + finally: + _libarchive.archive_read_close(a) + finally: + _libarchive.archive_read_free(a) + return [] + + +class EntryReadStream(object): + '''A file-like object for reading an entry from the archive.''' + def __init__(self, archive, size): + self.archive = archive + self.closed = False + self.size = size + self.bytes = 0 + + def __enter__(self): + return self + + def __exit__(self, *args): + return + + def __iter__(self): + if self.closed: + return + while True: + data = self.read(BLOCK_SIZE) + if not data: + break + yield data + + def __len__(self): + return self.size + + def tell(self): + return self.bytes + + def read(self, bytes=-1): + if self.closed: + return + if self.bytes == self.size: + # EOF already reached. + return + if bytes < 0: + bytes = self.size - self.bytes + elif self.bytes + bytes > self.size: + # Limit read to remaining bytes + bytes = self.size - self.bytes + # Read requested bytes + data = _libarchive.archive_read_data_into_str(self.archive._a, bytes) + self.bytes += len(data) + return data + + def close(self): + if self.closed: + return + # Call archive.close() with _defer True to let it know we have been + # closed and it is now safe to actually close. + self.archive.close(_defer=True) + self.archive = None + self.closed = True + + +class EntryWriteStream(object): + '''A file-like object for writing an entry to an archive. + + If the size is known ahead of time and provided, then the file contents + are not buffered but flushed directly to the archive. If size is omitted, + then the file contents are buffered and flushed in the close() method.''' + def __init__(self, archive, pathname, size=None): + self.archive = archive + self.entry = Entry(pathname=pathname, mtime=time.time(), mode=stat.S_IFREG) + if size is None: + self.buffer = six.StringIO() + else: + self.buffer = None + self.entry.size = size + self.entry.to_archive(self.archive) + self.bytes = 0 + self.closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def __del__(self): + self.close() + + def __len__(self): + return self.bytes + + def tell(self): + return self.bytes + + def write(self, data): + if self.closed: + raise Exception('Cannot write to closed stream.') + if self.buffer: + self.buffer.write(data) + else: + _libarchive.archive_write_data_from_str(self.archive._a, data) + self.bytes += len(data) + + def close(self): + if self.closed: + return + if self.buffer: + self.entry.size = self.buffer.tell() + self.entry.to_archive(self.archive) + _libarchive.archive_write_data_from_str(self.archive._a, self.buffer.getvalue()) + _libarchive.archive_write_finish_entry(self.archive._a) + + # Call archive.close() with _defer True to let it know we have been + # closed and it is now safe to actually close. + self.archive.close(_defer=True) + self.archive = None + self.closed = True + + +class Entry(object): + '''An entry within an archive. Represents the header data and it's location within the archive.''' + def __init__(self, pathname=None, size=None, mtime=None, mode=None, hpos=None, encoding=ENCODING): + self.pathname = pathname + self.size = size + self.mtime = mtime + self.mode = mode + self.hpos = hpos + self.encoding = encoding + self.linkname = None + self.id = None + self.hardlink = None + + @property + def header_position(self): + return self.hpos + + @classmethod + def from_archive(cls, archive, encoding=ENCODING): + '''Instantiates an Entry class and sets all the properties from an archive header.''' + e = _libarchive.archive_entry_new() + try: + call_and_check(_libarchive.archive_read_next_header2, archive._a, archive._a, e) + mode = _libarchive.archive_entry_filetype(e) + mode |= _libarchive.archive_entry_perm(e) + mtime = _libarchive.archive_entry_mtime(e) + _libarchive.archive_entry_mtime_nsec(e) / 1000000000.0 + # use current time as mtime if stored mtime is equal to 0 + mtime = mtime or time.time() + entry = cls( + pathname=_libarchive.archive_entry_pathname(e).decode(encoding), + size=_libarchive.archive_entry_size(e), + mtime=mtime, + mode=mode, + hpos=archive.header_position, + ) + # check hardlinkness first to processes hardlinks to the symlinks correctly + hardlink = _libarchive.archive_entry_hardlink(e) + if hardlink: + entry.hardlink = hardlink + elif entry.issym(): + entry.linkname = _libarchive.archive_entry_symlink(e) + finally: + _libarchive.archive_entry_free(e) + return entry + + @classmethod + def from_file(cls, f, entry=None, encoding=ENCODING, mtime=None): + '''Instantiates an Entry class and sets all the properties from a file on the file system. + f can be a file-like object or a path.''' + if entry is None: + entry = cls(encoding=encoding) + if entry.pathname is None: + if isinstance(f, six.string_types): + st = os.lstat(f) + entry.pathname = f + entry.size = st.st_size + entry.mtime = st.st_mtime if mtime is None else mtime + entry.mode = st.st_mode + entry.id = cls.get_entry_id(st) + if entry.issym(): + entry.linkname = os.readlink(f) + elif hasattr(f, 'fileno'): + st = os.fstat(f.fileno()) + entry.pathname = getattr(f, 'name', None) + entry.size = st.st_size + entry.mtime = st.st_mtime if mtime is None else mtime + entry.mode = st.st_mode + entry.id = cls.get_entry_id(st) + else: + entry.pathname = getattr(f, 'pathname', None) + entry.size = getattr(f, 'size', 0) + entry.mtime = getattr(f, 'mtime', time.time()) if mtime is None else mtime + entry.mode = getattr(f, 'mode', stat.S_IFREG) + return entry + + @staticmethod + def get_entry_id(st): + # windows doesn't have such information + if st.st_ino and st.st_dev: + return (st.st_dev, st.st_ino) + return None + + def to_archive(self, archive): + '''Creates an archive header and writes it to the given archive.''' + e = _libarchive.archive_entry_new() + try: + _libarchive.archive_entry_set_pathname(e, encode(self.pathname, self.encoding)) + _libarchive.archive_entry_set_filetype(e, stat.S_IFMT(self.mode)) + _libarchive.archive_entry_set_perm(e, stat.S_IMODE(self.mode)) + + nsec, sec = math.modf(self.mtime) + nsec *= 1000000000 + _libarchive.archive_entry_set_mtime(e, int(sec), int(nsec)) + + if self.ishardlink(): + _libarchive.archive_entry_set_size(e, 0) + _libarchive.archive_entry_set_hardlink(e, encode(self.hardlink, self.encoding)) + elif self.issym(): + _libarchive.archive_entry_set_size(e, 0) + _libarchive.archive_entry_set_symlink(e, encode(self.linkname, self.encoding)) + else: + _libarchive.archive_entry_set_size(e, self.size) + call_and_check(_libarchive.archive_write_header, archive._a, archive._a, e) + #self.hpos = archive.header_position + finally: + _libarchive.archive_entry_free(e) + + def isdir(self): + return stat.S_ISDIR(self.mode) + + def isfile(self): + return stat.S_ISREG(self.mode) + + def issym(self): + return stat.S_ISLNK(self.mode) + + def isfifo(self): + return stat.S_ISFIFO(self.mode) + + def ischr(self): + return stat.S_ISCHR(self.mode) + + def isblk(self): + return stat.S_ISBLK(self.mode) + + def ishardlink(self): + return bool(self.hardlink) + + +class Archive(object): + '''A low-level archive reader which provides forward-only iteration. Consider + this a light-weight pythonic libarchive wrapper.''' + def __init__(self, f, mode='rb', format=None, filter=None, entry_class=Entry, encoding=ENCODING, blocksize=BLOCK_SIZE, filter_opts=None, format_opts=None, fsync=False, fixed_mtime=None): + if six.PY2: + assert mode in ('r', 'rb', 'w', 'wb', 'a', 'ab'), 'Mode should be "r[b]", "w[b]" or "a[b]".' + else: + assert mode in ('rb', 'wb', 'ab'), 'Mode should be "rb", "wb", or "ab".' + self._stream = None + self.encoding = encoding + self.blocksize = blocksize + self.file_handle = None + self.fd = None + self.filename = None + self.fsync = fsync + if isinstance(f, six.string_types): + self.filename = f + self.file_handle = open(f, mode) + self.fd = self.file_handle.fileno() + # Only close it if we opened it... + self._defer_close = True + elif hasattr(f, 'fileno'): + self.filename = getattr(f, 'name', None) + self.file_handle = f + self.fd = self.file_handle.fileno() + # Leave the fd alone, caller should manage it... + self._defer_close = False + elif isinstance(f, int): + assert f >= 0, f + self.fd = f + # Leave the fd alone, caller should manage it... + self._defer_close = False + else: + raise Exception('Provided file is not path or open file.') + self.mode = mode + # Guess the format/filter from file name (if not provided) + if self.filename: + if format is None: + format = guess_format(self.filename)[0] + if filter is None: + filter = guess_format(self.filename)[1] + self.format = format + self.filter = filter + # The class to use for entries. + self.entry_class = entry_class + self.fixed_mtime = fixed_mtime + # Select filter/format functions. + if self.mode.startswith('r'): + self.format_func = get_func(self.format, FORMATS, 0) + if self.format_func is None: + raise Exception('Unsupported format %s' % format) + self.filter_func = get_func(self.filter, FILTERS, 0) + if self.filter_func is None: + raise Exception('Unsupported filter %s' % filter) + else: + # TODO: how to support appending? + if self.format is None: + raise Exception('You must specify a format for writing.') + self.format_func = get_func(self.format, FORMATS, 1) + if self.format_func is None: + raise Exception('Unsupported format %s' % format) + self.filter_func = get_func(self.filter, FILTERS, 1) + if self.filter_func is None: + raise Exception('Unsupported filter %s' % filter) + # Open the archive, apply filter/format functions. + self.filter_opts = filter_opts + self.format_opts = format_opts + # Stores every added entry's id to handle hardlinks properly + self.members = {} + self.init() + + def __iter__(self): + while True: + try: + yield self.entry_class.from_archive(self, encoding=self.encoding) + except EOF: + break + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __del__(self): + self.close() + + def init(self): + def _apply_opts(f, opts): + if opts: + for opt_name, opt_val in opts.items(): + call_and_check(f, self._a, self._a, None, encode(opt_name, self.encoding), encode(opt_val, self.encoding)) + + if self.mode.startswith('r'): + self._a = _libarchive.archive_read_new() + else: + self._a = _libarchive.archive_write_new() + self.format_func(self._a) + self.filter_func(self._a) + if self.mode.startswith('r'): + _apply_opts(_libarchive.archive_read_set_format_option, self.format_opts) + _apply_opts(_libarchive.archive_read_set_filter_option, self.filter_opts) + call_and_check(_libarchive.archive_read_open_fd, self._a, self._a, self.fd, self.blocksize) + else: + _apply_opts(_libarchive.archive_write_set_format_option, self.format_opts) + _apply_opts(_libarchive.archive_write_set_filter_option, self.filter_opts) + call_and_check(_libarchive.archive_write_open_fd, self._a, self._a, self.fd) + # XXX Don't pad the last block to avoid badly formed archive with zstd filter + call_and_check(_libarchive.archive_write_set_bytes_in_last_block, self._a, self._a, 1) + + def denit(self): + '''Closes and deallocates the archive reader/writer.''' + if getattr(self, '_a', None) is None: + return + try: + if self.mode.startswith('r'): + _libarchive.archive_read_close(self._a) + _libarchive.archive_read_free(self._a) + else: + _libarchive.archive_write_close(self._a) + _libarchive.archive_write_free(self._a) + finally: + # We only want one try at this... + self._a = None + + def close(self, _defer=False): + # _defer == True is how a stream can notify Archive that the stream is + # now closed. Calling it directly in not recommended. + if _defer: + # This call came from our open stream. + self._stream = None + if not self._defer_close: + # We are not yet ready to close. + return + if self._stream is not None: + # We have a stream open! don't close, but remember we were asked to. + self._defer_close = True + return + self.denit() + # If there is a file attached... + if getattr(self, 'file_handle', None): + # Make sure it is not already closed... + if getattr(self.file_handle, 'closed', False): + return + # Flush it if not read-only... + if not self.file_handle.mode.startswith('r'): + self.file_handle.flush() + if self.fsync: + os.fsync(self.fd) + # and then close it, if we opened it... + if getattr(self, 'close', None): + self.file_handle.close() + + @property + def header_position(self): + '''The position within the file.''' + return _libarchive.archive_read_header_position(self._a) + + def iterpaths(self): + for entry in self: + yield entry.pathname + + def read(self, size): + '''Read current archive entry contents into string.''' + return _libarchive.archive_read_data_into_str(self._a, size) + + def readpath(self, f): + '''Write current archive entry contents to file. f can be a file-like object or + a path.''' + with contextlib2.ExitStack() as exit_stack: + if isinstance(f, six.string_types): + basedir = os.path.basename(f) + if not os.path.exists(basedir): + os.makedirs(basedir) + f = exit_stack.enter_context(open(f, 'wb')) + return _libarchive.archive_read_data_into_fd(self._a, f.fileno()) + + def readstream(self, size): + '''Returns a file-like object for reading current archive entry contents.''' + self._stream = EntryReadStream(self, size) + return self._stream + + def write(self, member, data=None): + '''Writes a string buffer to the archive as the given entry.''' + if isinstance(member, six.string_types): + if self.fixed_mtime is None: + mtime = time.time() + else: + mtime = self.fixed_mtime + # Use default mode + member = self.entry_class(pathname=member, encoding=self.encoding, mtime=mtime, mode=stat.S_IFREG | 0o755) + if data: + member.size = len(data) + member.to_archive(self) + if data: + _libarchive.archive_write_data_from_str(self._a, data) + _libarchive.archive_write_finish_entry(self._a) + + def writepath(self, f, pathname=None): + '''Writes a file to the archive. f can be a file-like object or a path. Uses + write() to do the actual writing.''' + member = self.entry_class.from_file(f, encoding=self.encoding, mtime=self.fixed_mtime) + + with contextlib2.ExitStack() as exit_stack: + if isinstance(f, six.string_types): + if os.path.isfile(f): + f = exit_stack.enter_context(open(f, 'rb')) + if pathname: + member.pathname = pathname + + # hardlinks and symlink has no data to be written + if member.id in self.members: + member.hardlink = self.members[member.id] + self.write(member) + return + elif member.issym(): + self.write(member) + elif hasattr(f, 'read') and hasattr(f, 'seek') and hasattr(f, 'tell'): + self.write_from_file_object(member, f) + elif hasattr(f, 'read'): + # TODO: optimize this to write directly from f to archive. + self.write(member, data=f.read()) + else: + self.write(member) + + if member.id: + self.members[member.id] = member.pathname + + def write_from_file_object(self, member, fileobj): + if isinstance(member, six.string_types): + member = self.entry_class(pathname=member, encoding=self.encoding, mtime=self.fixed_mtime) + + start = fileobj.tell() + fileobj.seek(0, os.SEEK_END) + size = fileobj.tell() - start + fileobj.seek(start, os.SEEK_SET) + + if size: + member.size = size + member.to_archive(self) + + while size: + data = fileobj.read(BLOCK_SIZE) + if not data: + break + + size -= len(data) + if size < 0: + msg = "File ({}) size has changed. Can't write more data than was declared in the tar header ({}). " \ + "(probably file was changed during archiving)".format(member.pathname, member.size) + logger.warning(msg) + # write rest expected data (size is negative) + _libarchive.archive_write_data_from_str(self._a, data[:size]) + break + + _libarchive.archive_write_data_from_str(self._a, data) + + _libarchive.archive_write_finish_entry(self._a) + + def writestream(self, pathname, size=None): + '''Returns a file-like object for writing a new entry.''' + self._stream = EntryWriteStream(self, pathname, size) + return self._stream + + def printlist(self, s=sys.stdout): + for entry in self: + s.write(entry.size) + s.write('\t') + s.write(entry.mtime.strftime(MTIME_FORMAT)) + s.write('\t') + s.write(entry.pathname) + s.flush() + + +class SeekableArchive(Archive): + '''A class that provides random-access to archive entries. It does this by using one + or many Archive instances to seek to the correct location. The best performance will + occur when reading archive entries in the order in which they appear in the archive. + Reading out of order will cause the archive to be closed and opened each time a + reverse seek is needed.''' + def __init__(self, f, **kwargs): + self._stream = None + # Convert file to open file. We need this to reopen the archive. + mode = kwargs.setdefault('mode', 'rb') + if isinstance(f, six.string_types): + f = open(f, mode) + super(SeekableArchive, self).__init__(f, **kwargs) + self.entries = [] + self.eof = False + + def __iter__(self): + for entry in self.entries: + yield entry + if not self.eof: + try: + for entry in super(SeekableArchive, self).__iter__(): + self.entries.append(entry) + yield entry + except StopIteration: + self.eof = True + + def reopen(self): + '''Seeks the underlying fd to 0 position, then opens the archive. If the archive + is already open, this will effectively re-open it (rewind to the beginning).''' + self.denit() + self.file_handle.seek(0) + self.init() + + def getentry(self, pathname): + '''Take a name or entry object and returns an entry object.''' + for entry in self: + if entry.pathname == pathname: + return entry + raise KeyError(pathname) + + def seek(self, entry): + '''Seeks the archive to the requested entry. Will reopen if necessary.''' + move = entry.header_position - self.header_position + if move != 0: + if move < 0: + # can't move back, re-open archive: + self.reopen() + # move to proper position in stream + for curr in super(SeekableArchive, self).__iter__(): + if curr.header_position == entry.header_position: + break + + def read(self, member): + '''Return the requested archive entry contents as a string.''' + entry = self.getentry(member) + self.seek(entry) + return super(SeekableArchive, self).read(entry.size) + + def readpath(self, member, f): + entry = self.getentry(member) + self.seek(entry) + return super(SeekableArchive, self).readpath(f) + + def readstream(self, member): + '''Returns a file-like object for reading requested archive entry contents.''' + entry = self.getentry(member) + self.seek(entry) + self._stream = EntryReadStream(self, entry.size) + return self._stream diff --git a/contrib/python/python-libarchive/py2/libarchive/_libarchive.swg b/contrib/python/python-libarchive/py2/libarchive/_libarchive.swg new file mode 100644 index 00000000000..2fcb05420e4 --- /dev/null +++ b/contrib/python/python-libarchive/py2/libarchive/_libarchive.swg @@ -0,0 +1,339 @@ +/* Copyright (c) 2011, SmartFile <[email protected]> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the organization nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + +%module _libarchive + +%{ +#define SWIG_PYTHON_STRICT_BYTE_CHAR + +#include <archive.h> +#include <archive_entry.h> +%} + +%include "typemaps.i" + +%typemap(in) time_t +{ + if (PyLong_Check($input)) + $1 = (time_t) PyLong_AsLong($input); + else if (PyInt_Check($input)) + $1 = (time_t) PyInt_AsLong($input); + else if (PyFloat_Check($input)) + $1 = (time_t) PyFloat_AsDouble($input); + else { + PyErr_SetString(PyExc_TypeError,"Expected a large number"); + return NULL; + } +} + +%typemap(out) time_t +{ + $result = PyLong_FromLong((long)$1); +} + +%typemap(in) int64_t +{ + if (PyLong_Check($input)) + $1 = (int64_t) PyLong_AsLong($input); + else if (PyInt_Check($input)) + $1 = (int64_t) PyInt_AsLong($input); + else if (PyFloat_Check($input)) + $1 = (int64_t) PyFloat_AsDouble($input); + else { + PyErr_SetString(PyExc_TypeError,"Expected a large number"); + return NULL; + } +} + +%typemap(out) int64_t +{ + $result = PyLong_FromLong((long)$1); +} + +#define __LA_INT64_T long long +#define __LA_MODE_T int + +/* STRUCTURES */ +struct archive; +struct archive_entry; + +/* ARCHIVE READING */ +extern struct archive *archive_read_new(void); +extern int archive_read_free(struct archive *); + +/* opening */ +extern int archive_read_open_filename(struct archive *, + const char *_filename, size_t _block_size); +extern int archive_read_open_memory(struct archive *, + void * buff, size_t size); +extern int archive_read_open_memory2(struct archive *a, void *buff, + size_t size, size_t read_size); +extern int archive_read_open_fd(struct archive *, int _fd, + size_t _block_size); + +/* closing */ +extern int archive_read_close(struct archive *); +extern int archive_format(struct archive *); + +/* headers */ +extern int archive_read_next_header2(struct archive *, + struct archive_entry *); +extern const struct stat *archive_entry_stat(struct archive_entry *); +extern __LA_INT64_T archive_read_header_position(struct archive *); + +/* data */ +extern int archive_read_data_skip(struct archive *); +extern int archive_read_data_into_fd(struct archive *, int fd); + +/* FILTERS */ +extern int archive_read_support_filter_all(struct archive *); +extern int archive_read_support_filter_bzip2(struct archive *); +extern int archive_read_support_filter_compress(struct archive *); +extern int archive_read_support_filter_gzip(struct archive *); +extern int archive_read_support_filter_lzip(struct archive *); +extern int archive_read_support_filter_lzma(struct archive *); +extern int archive_read_support_filter_none(struct archive *); +extern int archive_read_support_filter_rpm(struct archive *); +extern int archive_read_support_filter_uu(struct archive *); +extern int archive_read_support_filter_xz(struct archive *); +extern int archive_read_support_filter_zstd(struct archive *); + +extern int archive_filter_count(struct archive *); +extern const char * archive_filter_name(struct archive *, int); + +/* FORMATS */ +extern int archive_read_support_format_all(struct archive *); +extern int archive_read_support_format_7zip(struct archive *); +extern int archive_read_support_format_ar(struct archive *); +extern int archive_read_support_format_cab(struct archive *); +extern int archive_read_support_format_cpio(struct archive *); +extern int archive_read_support_format_empty(struct archive *); +extern int archive_read_support_format_gnutar(struct archive *); +extern int archive_read_support_format_iso9660(struct archive *); +extern int archive_read_support_format_lha(struct archive *); +/*extern int archive_read_support_format_mtree(struct archive *);*/ +extern int archive_read_support_format_rar(struct archive *); +extern int archive_read_support_format_raw(struct archive *); +extern int archive_read_support_format_tar(struct archive *); +extern int archive_read_support_format_xar(struct archive *); +extern int archive_read_support_format_zip(struct archive *); +/*extern int archive_read_support_format_by_code(struct archive *, int);*/ + +/* OPTIONS */ +extern int archive_write_set_bytes_in_last_block(struct archive *_a, int bytes_in_last_block); +extern int archive_write_set_filter_option(struct archive *_a, const char *m, const char *o, const char *v); +extern int archive_write_zip_set_compression_deflate(struct archive *_a); +extern int archive_write_set_format_option(struct archive *_a, const char *m, const char *o, const char *v); +extern int archive_read_set_filter_option(struct archive *_a, const char *m, const char *o, const char *v); +extern int archive_read_set_format_option(struct archive *_a, const char *m, const char *o, const char *v); + +/* ARCHIVE WRITING */ +extern struct archive *archive_write_new(void); +extern int archive_write_free(struct archive *); + +/* opening */ +extern int archive_write_open(struct archive *, void *, + archive_open_callback *, archive_write_callback *, + archive_close_callback *); +extern int archive_write_open_fd(struct archive *, int _fd); +extern int archive_write_open_filename(struct archive *, const char *_file); +extern int archive_write_open_filename_w(struct archive *, + const wchar_t *_file); +extern int archive_write_open_memory(struct archive *, + void *_buffer, size_t _buffSize, size_t *_used); + +/* closing */ +extern int archive_write_close(struct archive *); + +/* headers */ +extern int archive_write_header(struct archive *, + struct archive_entry *); + +/* data */ + +/* commit */ +extern int archive_write_finish_entry(struct archive *); + +/* FILTERS */ +extern int archive_write_add_filter_bzip2(struct archive *); +extern int archive_write_add_filter_compress(struct archive *); +extern int archive_write_add_filter_gzip(struct archive *); +extern int archive_write_add_filter_lzip(struct archive *); +extern int archive_write_add_filter_lzma(struct archive *); +extern int archive_write_add_filter_none(struct archive *); +extern int archive_write_add_filter_xz(struct archive *); +extern int archive_write_add_filter_zstd(struct archive *); + + +/* FORMATS */ +/* A convenience function to set the format based on the code or name. */ +extern int archive_write_set_format(struct archive *, int format_code); +extern int archive_write_set_format_by_name(struct archive *, + const char *name); +/* To minimize link pollution, use one or more of the following. */ +extern int archive_write_set_format_ar_bsd(struct archive *); +extern int archive_write_set_format_ar_svr4(struct archive *); +extern int archive_write_set_format_cpio(struct archive *); +extern int archive_write_set_format_cpio_newc(struct archive *); +extern int archive_write_set_format_gnutar(struct archive *); +extern int archive_write_set_format_iso9660(struct archive *); +/*extern int archive_write_set_format_mtree(struct archive *);*/ +/* TODO: int archive_write_set_format_old_tar(struct archive *); */ +extern int archive_write_set_format_pax(struct archive *); +extern int archive_write_set_format_pax_restricted(struct archive *); +extern int archive_write_set_format_shar(struct archive *); +extern int archive_write_set_format_shar_dump(struct archive *); +extern int archive_write_set_format_ustar(struct archive *); +extern int archive_write_set_format_xar(struct archive *); +extern int archive_write_set_format_zip(struct archive *); + +/* ARCHIVE ENTRY */ +extern struct archive_entry *archive_entry_new(void); +extern void archive_entry_free(struct archive_entry *); +extern const char *archive_entry_symlink(struct archive_entry *); +extern void archive_entry_set_symlink(struct archive_entry *, const char *); +extern const char *archive_entry_hardlink(struct archive_entry *); +extern void archive_entry_set_hardlink(struct archive_entry *, const char *); + +/* ARCHIVE ENTRY PROPERTY ACCESS */ +/* reading */ +extern const char *archive_entry_pathname(struct archive_entry *); +extern const wchar_t *archive_entry_pathname_w(struct archive_entry *); +extern __LA_INT64_T archive_entry_size(struct archive_entry *); +extern time_t archive_entry_mtime(struct archive_entry *); +extern time_t archive_entry_mtime_nsec(struct archive_entry *); +extern __LA_MODE_T archive_entry_filetype(struct archive_entry *); +extern __LA_MODE_T archive_entry_perm(struct archive_entry *); + +/* writing */ +extern void archive_entry_set_pathname(struct archive_entry *, const char *); +extern void archive_entry_set_size(struct archive_entry *, __LA_INT64_T); +extern void archive_entry_set_mtime(struct archive_entry *, time_t, long); +extern void archive_entry_set_filetype(struct archive_entry *, unsigned int); +extern void archive_entry_set_perm(struct archive_entry *, __LA_MODE_T); + + +/* ERROR HANDLING */ +extern int archive_errno(struct archive *); +extern const char *archive_error_string(struct archive *); + + +/* CONSTANTS */ +#define ARCHIVE_VERSION_NUMBER 3000001 +#define ARCHIVE_VERSION_STRING "libarchive 3.0.1b" +#define ARCHIVE_EOF 1 /* Found end of archive. */ +#define ARCHIVE_OK 0 /* Operation was successful. */ +#define ARCHIVE_RETRY (-10) /* Retry might succeed. */ +#define ARCHIVE_WARN (-20) /* Partial success. */ +#define ARCHIVE_FAILED (-25) /* Current operation cannot complete. */ +#define ARCHIVE_FATAL (-30) /* No more operations are possible. */ + +#define ARCHIVE_FILTER_NONE 0 +#define ARCHIVE_FILTER_GZIP 1 +#define ARCHIVE_FILTER_BZIP2 2 +#define ARCHIVE_FILTER_COMPRESS 3 +#define ARCHIVE_FILTER_PROGRAM 4 +#define ARCHIVE_FILTER_LZMA 5 +#define ARCHIVE_FILTER_XZ 6 +#define ARCHIVE_FILTER_UU 7 +#define ARCHIVE_FILTER_RPM 8 +#define ARCHIVE_FILTER_LZIP 9 + +#define ARCHIVE_FORMAT_BASE_MASK 0xff0000 +#define ARCHIVE_FORMAT_CPIO 0x10000 +#define ARCHIVE_FORMAT_CPIO_POSIX (ARCHIVE_FORMAT_CPIO | 1) +#define ARCHIVE_FORMAT_CPIO_BIN_LE (ARCHIVE_FORMAT_CPIO | 2) +#define ARCHIVE_FORMAT_CPIO_BIN_BE (ARCHIVE_FORMAT_CPIO | 3) +#define ARCHIVE_FORMAT_CPIO_SVR4_NOCRC (ARCHIVE_FORMAT_CPIO | 4) +#define ARCHIVE_FORMAT_CPIO_SVR4_CRC (ARCHIVE_FORMAT_CPIO | 5) +#define ARCHIVE_FORMAT_CPIO_AFIO_LARGE (ARCHIVE_FORMAT_CPIO | 6) +#define ARCHIVE_FORMAT_SHAR 0x20000 +#define ARCHIVE_FORMAT_SHAR_BASE (ARCHIVE_FORMAT_SHAR | 1) +#define ARCHIVE_FORMAT_SHAR_DUMP (ARCHIVE_FORMAT_SHAR | 2) +#define ARCHIVE_FORMAT_TAR 0x30000 +#define ARCHIVE_FORMAT_TAR_USTAR (ARCHIVE_FORMAT_TAR | 1) +#define ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE (ARCHIVE_FORMAT_TAR | 2) +#define ARCHIVE_FORMAT_TAR_PAX_RESTRICTED (ARCHIVE_FORMAT_TAR | 3) +#define ARCHIVE_FORMAT_TAR_GNUTAR (ARCHIVE_FORMAT_TAR | 4) +#define ARCHIVE_FORMAT_ISO9660 0x40000 +#define ARCHIVE_FORMAT_ISO9660_ROCKRIDGE (ARCHIVE_FORMAT_ISO9660 | 1) +#define ARCHIVE_FORMAT_ZIP 0x50000 +#define ARCHIVE_FORMAT_EMPTY 0x60000 +#define ARCHIVE_FORMAT_AR 0x70000 +#define ARCHIVE_FORMAT_AR_GNU (ARCHIVE_FORMAT_AR | 1) +#define ARCHIVE_FORMAT_AR_BSD (ARCHIVE_FORMAT_AR | 2) +#define ARCHIVE_FORMAT_MTREE 0x80000 +#define ARCHIVE_FORMAT_RAW 0x90000 +#define ARCHIVE_FORMAT_XAR 0xA0000 +#define ARCHIVE_FORMAT_LHA 0xB0000 +#define ARCHIVE_FORMAT_CAB 0xC0000 +#define ARCHIVE_FORMAT_RAR 0xD0000 +#define ARCHIVE_FORMAT_7ZIP 0xE0000 + +#define ARCHIVE_EXTRACT_OWNER (0x0001) +#define ARCHIVE_EXTRACT_PERM (0x0002) +#define ARCHIVE_EXTRACT_TIME (0x0004) +#define ARCHIVE_EXTRACT_NO_OVERWRITE (0x0008) +#define ARCHIVE_EXTRACT_UNLINK (0x0010) +#define ARCHIVE_EXTRACT_ACL (0x0020) +#define ARCHIVE_EXTRACT_FFLAGS (0x0040) +#define ARCHIVE_EXTRACT_XATTR (0x0080) +#define ARCHIVE_EXTRACT_SECURE_SYMLINKS (0x0100) +#define ARCHIVE_EXTRACT_SECURE_NODOTDOT (0x0200) +#define ARCHIVE_EXTRACT_NO_AUTODIR (0x0400) +#define ARCHIVE_EXTRACT_NO_OVERWRITE_NEWER (0x0800) +#define ARCHIVE_EXTRACT_SPARSE (0x1000) +#define ARCHIVE_EXTRACT_MAC_METADATA (0x2000) + +%inline %{ +PyObject *archive_read_data_into_str(struct archive *archive, int len) { + PyObject *str = NULL; + if (!(str = PyBytes_FromStringAndSize(NULL, len))) { + PyErr_SetString(PyExc_MemoryError, "could not allocate string."); + return NULL; + } + if (len != archive_read_data(archive, PyBytes_AS_STRING(str), len)) { + PyErr_SetString(PyExc_RuntimeError, "could not read requested data."); + return NULL; + } + return str; +} + +PyObject *archive_write_data_from_str(struct archive *archive, PyObject *str) { + int len = PyBytes_Size(str); + if (len == 0) + return PyInt_FromLong(len); + int ret = archive_write_data(archive, PyBytes_AS_STRING(str), len); + if (ret == ARCHIVE_FATAL) { + PyErr_Format(PyExc_RuntimeError, "Could not write requested data - most likely no space left on device (error code: %d)", ret); + return NULL; + } + else if (ret <= 0) { + PyErr_Format(PyExc_RuntimeError, "Could not write requested data (error code: %d)", ret); + return NULL; + } + return PyInt_FromLong(len); +} +%} diff --git a/contrib/python/python-libarchive/py2/libarchive/tar.py b/contrib/python/python-libarchive/py2/libarchive/tar.py new file mode 100644 index 00000000000..f14149804b9 --- /dev/null +++ b/contrib/python/python-libarchive/py2/libarchive/tar.py @@ -0,0 +1,135 @@ +# Copyright (c) 2011, SmartFile <[email protected]> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the organization nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import time +from libarchive import is_archive, Entry, SeekableArchive +from tarfile import DEFAULT_FORMAT, USTAR_FORMAT, GNU_FORMAT, PAX_FORMAT, ENCODING +from tarfile import REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE + +FORMAT_CONVERSION = { + USTAR_FORMAT: 'tar', + GNU_FORMAT: 'gnu', + PAX_FORMAT: 'pax', +} + + +def is_tarfile(filename): + return is_archive(filename, formats=('tar', 'gnu', 'pax')) + + +def open(**kwargs): + return TarFile(**kwargs) + + +class TarInfo(Entry): + def __init__(self, name): + super(TarInfo, self).__init__(pathname=name) + + fromtarfile = Entry.from_archive + + def get_name(self): + return self.pathname + + def set_name(self, value): + self.pathname = value + + name = property(get_name, set_name) + + @property + def get_type(self): + for attr, type in ( + ('isdir', DIRTYPE), ('isfile', REGTYPE), ('issym', SYMTYPE), + ('isfifo', FIFOTYPE), ('ischr', CHRTYPE), ('isblk', BLKTYPE), + ): + if getattr(self, attr)(): + return type + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + pax_headers = property(_get_missing, _set_missing) + + +class TarFile(SeekableArchive): + def __init__(self, name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, encoding=ENCODING): + if name: + f = name + elif fileobj: + f = fileobj + try: + format = FORMAT_CONVERSON.get(format) + except KeyError: + raise Exception('Invalid tar format: %s' % format) + super(TarFile, self).__init__(f, mode=mode, format=format, entry_class=tarinfo, encoding=encoding) + + getmember = SeekableArchive.getentry + list = SeekableArchive.printlist + extract = SeekableArchive.readpath + extractfile = SeekableArchive.readstream + + def getmembers(self): + return list(self) + + def getnames(self): + return list(self.iterpaths) + + def next(self): + pass # TODO: how to do this? + + def extract(self, member, path=None): + if path is None: + path = os.getcwd() + if isinstance(member, basestring): + f = os.path.join(path, member) + else: + f = os.path.join(path, member.pathname) + return self.readpath(member, f) + + def add(self, name, arcname, recursive=True, exclude=None, filter=None): + pass # TODO: implement this. + + def addfile(tarinfo, fileobj): + return self.writepath(fileobj, tarinfo) + + def gettarinfo(name=None, arcname=None, fileobj=None): + if name: + f = name + elif fileobj: + f = fileobj + entry = self.entry_class.from_file(f) + if arcname: + entry.pathname = arcname + return entry + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + pax_headers = property(_get_missing, _set_missing) diff --git a/contrib/python/python-libarchive/py2/libarchive/zip.py b/contrib/python/python-libarchive/py2/libarchive/zip.py new file mode 100644 index 00000000000..539f6dbcc49 --- /dev/null +++ b/contrib/python/python-libarchive/py2/libarchive/zip.py @@ -0,0 +1,151 @@ +# Copyright (c) 2011, SmartFile <[email protected]> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the organization nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os, time +from libarchive import is_archive, Entry, SeekableArchive +from zipfile import ZIP_STORED, ZIP_DEFLATED + + +def is_zipfile(filename): + return is_archive(filename, formats=('zip', )) + + +class ZipEntry(Entry): + def __init__(self, *args, **kwargs): + super(ZipEntry, self).__init__(*args, **kwargs) + + def get_filename(self): + return self.pathname + + def set_filename(self, value): + self.pathname = value + + filename = property(get_filename, set_filename) + + def get_file_size(self): + return self.size + + def set_file_size(self, value): + assert isinstance(size, (int, long)), 'Please provide size as int or long.' + self.size = value + + file_size = property(get_file_size, set_file_size) + + def get_date_time(self): + return time.localtime(self.mtime)[0:6] + + def set_date_time(self, value): + assert isinstance(value, tuple), 'mtime should be tuple (year, month, day, hour, minute, second).' + assert len(value) == 6, 'mtime should be tuple (year, month, day, hour, minute, second).' + self.mtime = time.mktime(value + (0, 0, 0)) + + date_time = property(get_date_time, set_date_time) + + header_offset = Entry.header_position + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + compress_type = property(_get_missing, _set_missing) + comment = property(_get_missing, _set_missing) + extra = property(_get_missing, _set_missing) + create_system = property(_get_missing, _set_missing) + create_version = property(_get_missing, _set_missing) + extract_version = property(_get_missing, _set_missing) + reserved = property(_get_missing, _set_missing) + flag_bits = property(_get_missing, _set_missing) + volume = property(_get_missing, _set_missing) + internal_attr = property(_get_missing, _set_missing) + external_attr = property(_get_missing, _set_missing) + CRC = property(_get_missing, _set_missing) + compress_size = property(_get_missing, _set_missing) + + +class ZipFile(SeekableArchive): + def __init__(self, f, mode='r', compression=ZIP_DEFLATED, allowZip64=False): + super(ZipFile, self).__init__(f, mode=mode, format='zip', entry_class=ZipEntry, encoding='CP437') + if mode == 'w' and compression == ZIP_STORED: + # Disable compression for writing. + _libarchive.archive_write_set_format_option(self.archive._a, "zip", "compression", "store") + self.compression = compression + + getinfo = SeekableArchive.getentry + + def namelist(self): + return list(self.iterpaths) + + def infolist(self): + return list(self) + + def open(self, name, mode, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + if mode == 'r': + return self.readstream(name) + else: + return self.writestream(name) + + def extract(self, name, path=None, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + if not path: + path = os.getcwd() + return self.readpath(name, os.path.join(path, name)) + + def extractall(self, path, names=None, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + if not names: + names = self.namelist() + if names: + for name in names: + self.extract(name, path) + + def read(self, name, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + return self.read(name) + + def writestr(self, member, data, compress_type=None): + if compress_type != self.compression: + raise Exception('Cannot change compression type for individual entries.') + return self.write(member, data) + + def setpassword(self, pwd): + raise NotImplemented('Encryption not supported.') + + def testzip(self): + raise NotImplemented() + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + comment = property(_get_missing, _set_missing) diff --git a/contrib/python/python-libarchive/py2/ya.make b/contrib/python/python-libarchive/py2/ya.make new file mode 100644 index 00000000000..8902a10c527 --- /dev/null +++ b/contrib/python/python-libarchive/py2/ya.make @@ -0,0 +1,30 @@ +PY2_LIBRARY() + +LICENSE(BSD-3-Clause) + +SUBSCRIBER(g:python-contrib) + +VERSION(3.1.2.post1) + +PEERDIR( + contrib/libs/libarchive + contrib/python/contextlib2 + contrib/python/six +) + +ADDINCL( + contrib/libs/libarchive/libarchive +) + +NO_LINT() + +PY_SRCS( + SWIG_C + TOP_LEVEL + libarchive/__init__.py + libarchive/tar.py + libarchive/zip.py + libarchive/_libarchive.swg +) + +END() diff --git a/contrib/python/python-libarchive/py3/libarchive/__init__.py b/contrib/python/python-libarchive/py3/libarchive/__init__.py new file mode 100644 index 00000000000..0c0c63359a3 --- /dev/null +++ b/contrib/python/python-libarchive/py3/libarchive/__init__.py @@ -0,0 +1,800 @@ +# Copyright (c) 2011, SmartFile <[email protected]> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the organization nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import stat +import sys +import math +import time +import logging +import warnings + +import contextlib2 + +from libarchive import _libarchive +import six + +logger = logging.getLogger(__name__) + +# Suggested block size for libarchive. Libarchive may adjust it. +BLOCK_SIZE = 10240 + +MTIME_FORMAT = '' + +# Default encoding scheme. +ENCODING = 'utf-8' + +if six.PY2: + def encode(value, encoding): + if type(value) == str: + value = value.decode(encoding, errors='ignore') + return value.encode(encoding) +else: + def encode(value, encoding): + return value.encode(encoding) + + +# Functions to initialize read/write for various libarchive supported formats and filters. +FORMATS = { + None: (_libarchive.archive_read_support_format_all, None), + 'tar': (_libarchive.archive_read_support_format_tar, _libarchive.archive_write_set_format_ustar), + 'pax': (_libarchive.archive_read_support_format_tar, _libarchive.archive_write_set_format_pax), + 'gnu': (_libarchive.archive_read_support_format_gnutar, _libarchive.archive_write_set_format_gnutar), + 'zip': (_libarchive.archive_read_support_format_zip, _libarchive.archive_write_set_format_zip), + 'rar': (_libarchive.archive_read_support_format_rar, None), + '7zip': (_libarchive.archive_read_support_format_7zip, None), + 'ar': (_libarchive.archive_read_support_format_ar, None), + 'cab': (_libarchive.archive_read_support_format_cab, None), + 'cpio': (_libarchive.archive_read_support_format_cpio, _libarchive.archive_write_set_format_cpio_newc), + 'iso': (_libarchive.archive_read_support_format_iso9660, _libarchive.archive_write_set_format_iso9660), + 'lha': (_libarchive.archive_read_support_format_lha, None), + 'xar': (_libarchive.archive_read_support_format_xar, _libarchive.archive_write_set_format_xar), +} + +FILTERS = { + None: (_libarchive.archive_read_support_filter_all, _libarchive.archive_write_add_filter_none), + 'bzip2': (_libarchive.archive_read_support_filter_bzip2, _libarchive.archive_write_add_filter_bzip2), + 'gzip': (_libarchive.archive_read_support_filter_gzip, _libarchive.archive_write_add_filter_gzip), + 'zstd': (_libarchive.archive_read_support_filter_zstd, _libarchive.archive_write_add_filter_zstd), +} + +# Map file extensions to formats and filters. To support quick detection. +FORMAT_EXTENSIONS = { + '.tar': 'tar', + '.zip': 'zip', + '.rar': 'rar', + '.7z': '7zip', + '.ar': 'ar', + '.cab': 'cab', + '.rpm': 'cpio', + '.cpio': 'cpio', + '.iso': 'iso', + '.lha': 'lha', + '.xar': 'xar', +} +FILTER_EXTENSIONS = { + '.bz2': 'bzip2', + '.gz': 'gzip', + '.zst': 'zstd', +} + + +class EOF(Exception): + '''Raised by ArchiveInfo.from_archive() when unable to read the next + archive header.''' + pass + + +def get_error(archive): + '''Retrieves the last error description for the given archive instance.''' + return _libarchive.archive_error_string(archive) + + +def call_and_check(func, archive, *args): + '''Executes a libarchive function and raises an exception when appropriate.''' + ret = func(*args) + if ret == _libarchive.ARCHIVE_OK: + return + elif ret == _libarchive.ARCHIVE_WARN: + warnings.warn('Warning executing function: %s.' % get_error(archive), RuntimeWarning) + elif ret == _libarchive.ARCHIVE_EOF: + raise EOF() + else: + raise Exception('Fatal error executing function, message is: %s.' % get_error(archive)) + + +def get_func(name, items, index): + item = items.get(name, None) + if item is None: + return None + return item[index] + + +def guess_format(filename): + filename, ext = os.path.splitext(filename) + filter = FILTER_EXTENSIONS.get(ext) + if filter: + filename, ext = os.path.splitext(filename) + format = FORMAT_EXTENSIONS.get(ext) + return format, filter + + +def is_archive_name(filename, formats=None): + '''Quick check to see if the given file has an extension indiciating that it is + an archive. The format parameter can be used to limit what archive format is acceptable. + If omitted, all supported archive formats will be checked. + + This function will return the name of the most likely archive format, None if the file is + unlikely to be an archive.''' + if formats is None: + formats = FORMAT_EXTENSIONS.values() + format, filter = guess_format(filename) + if format in formats: + return format + + +def is_archive(f, formats=(None, ), filters=(None, )): + '''Check to see if the given file is actually an archive. The format parameter + can be used to specify which archive format is acceptable. If ommitted, all supported + archive formats will be checked. It opens the file using libarchive. If no error is + received, the file was successfully detected by the libarchive bidding process. + + This procedure is quite costly, so you should avoid calling it unless you are reasonably + sure that the given file is an archive. In other words, you may wish to filter large + numbers of file names using is_archive_name() before double-checking the positives with + this function. + + This function will return True if the file can be opened as an archive using the given + format(s)/filter(s).''' + with contextlib2.ExitStack() as exit_stack: + if isinstance(f, six.string_types): + f = exit_stack.enter_context(open(f, 'rb')) + a = _libarchive.archive_read_new() + for format in formats: + format = get_func(format, FORMATS, 0) + if format is None: + return False + format(a) + for filter in filters: + filter = get_func(filter, FILTERS, 0) + if filter is None: + return False + filter(a) + try: + try: + call_and_check(_libarchive.archive_read_open_fd, a, a, f.fileno(), BLOCK_SIZE) + return True + except: + return False + finally: + _libarchive.archive_read_close(a) + _libarchive.archive_read_free(a) + + +def get_archive_filter_names(filename): + with open(filename, 'rb') as afile: + a = _libarchive.archive_read_new() + try: + format_func = get_func(None, FORMATS, 0) + format_func(a) + filter_func = get_func(None, FILTERS, 0) + filter_func(a) + if _libarchive.archive_read_open_fd(a, afile.fileno(), BLOCK_SIZE) == _libarchive.ARCHIVE_OK: + try: + nfilter = _libarchive.archive_filter_count(a) + return [_libarchive.archive_filter_name(a, i).decode(ENCODING) for i in range(nfilter)] + finally: + _libarchive.archive_read_close(a) + finally: + _libarchive.archive_read_free(a) + return [] + + +class EntryReadStream(object): + '''A file-like object for reading an entry from the archive.''' + def __init__(self, archive, size): + self.archive = archive + self.closed = False + self.size = size + self.bytes = 0 + + def __enter__(self): + return self + + def __exit__(self, *args): + return + + def __iter__(self): + if self.closed: + return + while True: + data = self.read(BLOCK_SIZE) + if not data: + break + yield data + + def __len__(self): + return self.size + + def tell(self): + return self.bytes + + def read(self, bytes=-1): + if self.closed: + return + if self.bytes == self.size: + # EOF already reached. + return + if bytes < 0: + bytes = self.size - self.bytes + elif self.bytes + bytes > self.size: + # Limit read to remaining bytes + bytes = self.size - self.bytes + # Read requested bytes + data = _libarchive.archive_read_data_into_str(self.archive._a, bytes) + self.bytes += len(data) + return data + + def close(self): + if self.closed: + return + # Call archive.close() with _defer True to let it know we have been + # closed and it is now safe to actually close. + self.archive.close(_defer=True) + self.archive = None + self.closed = True + + +class EntryWriteStream(object): + '''A file-like object for writing an entry to an archive. + + If the size is known ahead of time and provided, then the file contents + are not buffered but flushed directly to the archive. If size is omitted, + then the file contents are buffered and flushed in the close() method.''' + def __init__(self, archive, pathname, size=None): + self.archive = archive + self.entry = Entry(pathname=pathname, mtime=time.time(), mode=stat.S_IFREG) + if size is None: + self.buffer = six.StringIO() + else: + self.buffer = None + self.entry.size = size + self.entry.to_archive(self.archive) + self.bytes = 0 + self.closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def __del__(self): + self.close() + + def __len__(self): + return self.bytes + + def tell(self): + return self.bytes + + def write(self, data): + if self.closed: + raise Exception('Cannot write to closed stream.') + if self.buffer: + self.buffer.write(data) + else: + _libarchive.archive_write_data_from_str(self.archive._a, data) + self.bytes += len(data) + + def close(self): + if self.closed: + return + if self.buffer: + self.entry.size = self.buffer.tell() + self.entry.to_archive(self.archive) + _libarchive.archive_write_data_from_str(self.archive._a, self.buffer.getvalue()) + _libarchive.archive_write_finish_entry(self.archive._a) + + # Call archive.close() with _defer True to let it know we have been + # closed and it is now safe to actually close. + self.archive.close(_defer=True) + self.archive = None + self.closed = True + + +class Entry(object): + '''An entry within an archive. Represents the header data and it's location within the archive.''' + def __init__(self, pathname=None, size=None, mtime=None, mode=None, hpos=None, encoding=ENCODING): + self.pathname = pathname + self.size = size + self.mtime = mtime + self.mode = mode + self.hpos = hpos + self.encoding = encoding + self.linkname = None + self.id = None + self.hardlink = None + + @property + def header_position(self): + return self.hpos + + @classmethod + def from_archive(cls, archive, encoding=ENCODING): + '''Instantiates an Entry class and sets all the properties from an archive header.''' + e = _libarchive.archive_entry_new() + try: + call_and_check(_libarchive.archive_read_next_header2, archive._a, archive._a, e) + mode = _libarchive.archive_entry_filetype(e) + mode |= _libarchive.archive_entry_perm(e) + mtime = _libarchive.archive_entry_mtime(e) + _libarchive.archive_entry_mtime_nsec(e) / 1000000000.0 + # use current time as mtime if stored mtime is equal to 0 + mtime = mtime or time.time() + entry = cls( + pathname=_libarchive.archive_entry_pathname(e).decode(encoding), + size=_libarchive.archive_entry_size(e), + mtime=mtime, + mode=mode, + hpos=archive.header_position, + ) + # check hardlinkness first to processes hardlinks to the symlinks correctly + hardlink = _libarchive.archive_entry_hardlink(e) + if hardlink: + entry.hardlink = hardlink + elif entry.issym(): + entry.linkname = _libarchive.archive_entry_symlink(e) + finally: + _libarchive.archive_entry_free(e) + return entry + + @classmethod + def from_file(cls, f, entry=None, encoding=ENCODING, mtime=None): + '''Instantiates an Entry class and sets all the properties from a file on the file system. + f can be a file-like object or a path.''' + if entry is None: + entry = cls(encoding=encoding) + if entry.pathname is None: + if isinstance(f, six.string_types): + st = os.lstat(f) + entry.pathname = f + entry.size = st.st_size + entry.mtime = st.st_mtime if mtime is None else mtime + entry.mode = st.st_mode + entry.id = cls.get_entry_id(st) + if entry.issym(): + entry.linkname = os.readlink(f) + elif hasattr(f, 'fileno'): + st = os.fstat(f.fileno()) + entry.pathname = getattr(f, 'name', None) + entry.size = st.st_size + entry.mtime = st.st_mtime if mtime is None else mtime + entry.mode = st.st_mode + entry.id = cls.get_entry_id(st) + else: + entry.pathname = getattr(f, 'pathname', None) + entry.size = getattr(f, 'size', 0) + entry.mtime = getattr(f, 'mtime', time.time()) if mtime is None else mtime + entry.mode = getattr(f, 'mode', stat.S_IFREG) + return entry + + @staticmethod + def get_entry_id(st): + # windows doesn't have such information + if st.st_ino and st.st_dev: + return (st.st_dev, st.st_ino) + return None + + def to_archive(self, archive): + '''Creates an archive header and writes it to the given archive.''' + e = _libarchive.archive_entry_new() + try: + _libarchive.archive_entry_set_pathname(e, encode(self.pathname, self.encoding)) + _libarchive.archive_entry_set_filetype(e, stat.S_IFMT(self.mode)) + _libarchive.archive_entry_set_perm(e, stat.S_IMODE(self.mode)) + + nsec, sec = math.modf(self.mtime) + nsec *= 1000000000 + _libarchive.archive_entry_set_mtime(e, int(sec), int(nsec)) + + if self.ishardlink(): + _libarchive.archive_entry_set_size(e, 0) + _libarchive.archive_entry_set_hardlink(e, encode(self.hardlink, self.encoding)) + elif self.issym(): + _libarchive.archive_entry_set_size(e, 0) + _libarchive.archive_entry_set_symlink(e, encode(self.linkname, self.encoding)) + else: + _libarchive.archive_entry_set_size(e, self.size) + call_and_check(_libarchive.archive_write_header, archive._a, archive._a, e) + #self.hpos = archive.header_position + finally: + _libarchive.archive_entry_free(e) + + def isdir(self): + return stat.S_ISDIR(self.mode) + + def isfile(self): + return stat.S_ISREG(self.mode) + + def issym(self): + return stat.S_ISLNK(self.mode) + + def isfifo(self): + return stat.S_ISFIFO(self.mode) + + def ischr(self): + return stat.S_ISCHR(self.mode) + + def isblk(self): + return stat.S_ISBLK(self.mode) + + def ishardlink(self): + return bool(self.hardlink) + + +class Archive(object): + '''A low-level archive reader which provides forward-only iteration. Consider + this a light-weight pythonic libarchive wrapper.''' + def __init__(self, f, mode='rb', format=None, filter=None, entry_class=Entry, encoding=ENCODING, blocksize=BLOCK_SIZE, filter_opts=None, format_opts=None, fsync=False, fixed_mtime=None): + if six.PY2: + assert mode in ('r', 'rb', 'w', 'wb', 'a', 'ab'), 'Mode should be "r[b]", "w[b]" or "a[b]".' + else: + assert mode in ('rb', 'wb', 'ab'), 'Mode should be "rb", "wb", or "ab".' + self._stream = None + self.encoding = encoding + self.blocksize = blocksize + self.file_handle = None + self.fd = None + self.filename = None + self.fsync = fsync + if isinstance(f, six.string_types): + self.filename = f + self.file_handle = open(f, mode) + self.fd = self.file_handle.fileno() + # Only close it if we opened it... + self._defer_close = True + elif hasattr(f, 'fileno'): + self.filename = getattr(f, 'name', None) + self.file_handle = f + self.fd = self.file_handle.fileno() + # Leave the fd alone, caller should manage it... + self._defer_close = False + elif isinstance(f, int): + assert f >= 0, f + self.fd = f + # Leave the fd alone, caller should manage it... + self._defer_close = False + else: + raise Exception('Provided file is not path or open file.') + self.mode = mode + # Guess the format/filter from file name (if not provided) + if self.filename: + if format is None: + format = guess_format(self.filename)[0] + if filter is None: + filter = guess_format(self.filename)[1] + self.format = format + self.filter = filter + # The class to use for entries. + self.entry_class = entry_class + self.fixed_mtime = fixed_mtime + # Select filter/format functions. + if self.mode.startswith('r'): + self.format_func = get_func(self.format, FORMATS, 0) + if self.format_func is None: + raise Exception('Unsupported format %s' % format) + self.filter_func = get_func(self.filter, FILTERS, 0) + if self.filter_func is None: + raise Exception('Unsupported filter %s' % filter) + else: + # TODO: how to support appending? + if self.format is None: + raise Exception('You must specify a format for writing.') + self.format_func = get_func(self.format, FORMATS, 1) + if self.format_func is None: + raise Exception('Unsupported format %s' % format) + self.filter_func = get_func(self.filter, FILTERS, 1) + if self.filter_func is None: + raise Exception('Unsupported filter %s' % filter) + # Open the archive, apply filter/format functions. + self.filter_opts = filter_opts + self.format_opts = format_opts + # Stores every added entry's id to handle hardlinks properly + self.members = {} + self.init() + + def __iter__(self): + while True: + try: + yield self.entry_class.from_archive(self, encoding=self.encoding) + except EOF: + break + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __del__(self): + self.close() + + def init(self): + def _apply_opts(f, opts): + if opts: + for opt_name, opt_val in opts.items(): + call_and_check(f, self._a, self._a, None, encode(opt_name, self.encoding), encode(opt_val, self.encoding)) + + if self.mode.startswith('r'): + self._a = _libarchive.archive_read_new() + else: + self._a = _libarchive.archive_write_new() + self.format_func(self._a) + self.filter_func(self._a) + if self.mode.startswith('r'): + _apply_opts(_libarchive.archive_read_set_format_option, self.format_opts) + _apply_opts(_libarchive.archive_read_set_filter_option, self.filter_opts) + call_and_check(_libarchive.archive_read_open_fd, self._a, self._a, self.fd, self.blocksize) + else: + _apply_opts(_libarchive.archive_write_set_format_option, self.format_opts) + _apply_opts(_libarchive.archive_write_set_filter_option, self.filter_opts) + call_and_check(_libarchive.archive_write_open_fd, self._a, self._a, self.fd) + # XXX Don't pad the last block to avoid badly formed archive with zstd filter + call_and_check(_libarchive.archive_write_set_bytes_in_last_block, self._a, self._a, 1) + + def denit(self): + '''Closes and deallocates the archive reader/writer.''' + if getattr(self, '_a', None) is None: + return + try: + if self.mode.startswith('r'): + _libarchive.archive_read_close(self._a) + _libarchive.archive_read_free(self._a) + else: + _libarchive.archive_write_close(self._a) + _libarchive.archive_write_free(self._a) + finally: + # We only want one try at this... + self._a = None + + def close(self, _defer=False): + # _defer == True is how a stream can notify Archive that the stream is + # now closed. Calling it directly in not recommended. + if _defer: + # This call came from our open stream. + self._stream = None + if not self._defer_close: + # We are not yet ready to close. + return + if self._stream is not None: + # We have a stream open! don't close, but remember we were asked to. + self._defer_close = True + return + self.denit() + # If there is a file attached... + if getattr(self, 'file_handle', None): + # Make sure it is not already closed... + if getattr(self.file_handle, 'closed', False): + return + # Flush it if not read-only... + if not self.file_handle.mode.startswith('r'): + self.file_handle.flush() + if self.fsync: + os.fsync(self.fd) + # and then close it, if we opened it... + if getattr(self, 'close', None): + self.file_handle.close() + + @property + def header_position(self): + '''The position within the file.''' + return _libarchive.archive_read_header_position(self._a) + + def iterpaths(self): + for entry in self: + yield entry.pathname + + def read(self, size): + '''Read current archive entry contents into string.''' + return _libarchive.archive_read_data_into_str(self._a, size) + + def readpath(self, f): + '''Write current archive entry contents to file. f can be a file-like object or + a path.''' + with contextlib2.ExitStack() as exit_stack: + if isinstance(f, six.string_types): + basedir = os.path.basename(f) + if not os.path.exists(basedir): + os.makedirs(basedir) + f = exit_stack.enter_context(open(f, 'wb')) + return _libarchive.archive_read_data_into_fd(self._a, f.fileno()) + + def readstream(self, size): + '''Returns a file-like object for reading current archive entry contents.''' + self._stream = EntryReadStream(self, size) + return self._stream + + def write(self, member, data=None): + '''Writes a string buffer to the archive as the given entry.''' + if isinstance(member, six.string_types): + if self.fixed_mtime is None: + mtime = time.time() + else: + mtime = self.fixed_mtime + # Use default mode + member = self.entry_class(pathname=member, encoding=self.encoding, mtime=mtime, mode=stat.S_IFREG | 0o755) + if data: + member.size = len(data) + member.to_archive(self) + if data: + _libarchive.archive_write_data_from_str(self._a, data) + _libarchive.archive_write_finish_entry(self._a) + + def writepath(self, f, pathname=None): + '''Writes a file to the archive. f can be a file-like object or a path. Uses + write() to do the actual writing.''' + member = self.entry_class.from_file(f, encoding=self.encoding, mtime=self.fixed_mtime) + + with contextlib2.ExitStack() as exit_stack: + if isinstance(f, six.string_types): + if os.path.isfile(f): + f = exit_stack.enter_context(open(f, 'rb')) + if pathname: + member.pathname = pathname + + # hardlinks and symlink has no data to be written + if member.id in self.members: + member.hardlink = self.members[member.id] + self.write(member) + return + elif member.issym(): + self.write(member) + elif hasattr(f, 'read') and hasattr(f, 'seek') and hasattr(f, 'tell'): + self.write_from_file_object(member, f) + elif hasattr(f, 'read'): + # TODO: optimize this to write directly from f to archive. + self.write(member, data=f.read()) + else: + self.write(member) + + if member.id: + self.members[member.id] = member.pathname + + def write_from_file_object(self, member, fileobj): + if isinstance(member, six.string_types): + member = self.entry_class(pathname=member, encoding=self.encoding, mtime=self.fixed_mtime) + + start = fileobj.tell() + fileobj.seek(0, os.SEEK_END) + size = fileobj.tell() - start + fileobj.seek(start, os.SEEK_SET) + + if size: + member.size = size + member.to_archive(self) + + while size: + data = fileobj.read(BLOCK_SIZE) + if not data: + break + + size -= len(data) + if size < 0: + msg = "File ({}) size has changed. Can't write more data than was declared in the tar header ({}). " \ + "(probably file was changed during archiving)".format(member.pathname, member.size) + logger.warning(msg) + # write rest expected data (size is negative) + _libarchive.archive_write_data_from_str(self._a, data[:size]) + break + + _libarchive.archive_write_data_from_str(self._a, data) + + _libarchive.archive_write_finish_entry(self._a) + + def writestream(self, pathname, size=None): + '''Returns a file-like object for writing a new entry.''' + self._stream = EntryWriteStream(self, pathname, size) + return self._stream + + def printlist(self, s=sys.stdout): + for entry in self: + s.write(entry.size) + s.write('\t') + s.write(entry.mtime.strftime(MTIME_FORMAT)) + s.write('\t') + s.write(entry.pathname) + s.flush() + + +class SeekableArchive(Archive): + '''A class that provides random-access to archive entries. It does this by using one + or many Archive instances to seek to the correct location. The best performance will + occur when reading archive entries in the order in which they appear in the archive. + Reading out of order will cause the archive to be closed and opened each time a + reverse seek is needed.''' + def __init__(self, f, **kwargs): + self._stream = None + # Convert file to open file. We need this to reopen the archive. + mode = kwargs.setdefault('mode', 'rb') + if isinstance(f, six.string_types): + f = open(f, mode) + super(SeekableArchive, self).__init__(f, **kwargs) + self.entries = [] + self.eof = False + + def __iter__(self): + for entry in self.entries: + yield entry + if not self.eof: + try: + for entry in super(SeekableArchive, self).__iter__(): + self.entries.append(entry) + yield entry + except StopIteration: + self.eof = True + + def reopen(self): + '''Seeks the underlying fd to 0 position, then opens the archive. If the archive + is already open, this will effectively re-open it (rewind to the beginning).''' + self.denit() + self.file_handle.seek(0) + self.init() + + def getentry(self, pathname): + '''Take a name or entry object and returns an entry object.''' + for entry in self: + if entry.pathname == pathname: + return entry + raise KeyError(pathname) + + def seek(self, entry): + '''Seeks the archive to the requested entry. Will reopen if necessary.''' + move = entry.header_position - self.header_position + if move != 0: + if move < 0: + # can't move back, re-open archive: + self.reopen() + # move to proper position in stream + for curr in super(SeekableArchive, self).__iter__(): + if curr.header_position == entry.header_position: + break + + def read(self, member): + '''Return the requested archive entry contents as a string.''' + entry = self.getentry(member) + self.seek(entry) + return super(SeekableArchive, self).read(entry.size) + + def readpath(self, member, f): + entry = self.getentry(member) + self.seek(entry) + return super(SeekableArchive, self).readpath(f) + + def readstream(self, member): + '''Returns a file-like object for reading requested archive entry contents.''' + entry = self.getentry(member) + self.seek(entry) + self._stream = EntryReadStream(self, entry.size) + return self._stream diff --git a/contrib/python/python-libarchive/py3/libarchive/_libarchive.swg b/contrib/python/python-libarchive/py3/libarchive/_libarchive.swg new file mode 100644 index 00000000000..2fcb05420e4 --- /dev/null +++ b/contrib/python/python-libarchive/py3/libarchive/_libarchive.swg @@ -0,0 +1,339 @@ +/* Copyright (c) 2011, SmartFile <[email protected]> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the organization nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + +%module _libarchive + +%{ +#define SWIG_PYTHON_STRICT_BYTE_CHAR + +#include <archive.h> +#include <archive_entry.h> +%} + +%include "typemaps.i" + +%typemap(in) time_t +{ + if (PyLong_Check($input)) + $1 = (time_t) PyLong_AsLong($input); + else if (PyInt_Check($input)) + $1 = (time_t) PyInt_AsLong($input); + else if (PyFloat_Check($input)) + $1 = (time_t) PyFloat_AsDouble($input); + else { + PyErr_SetString(PyExc_TypeError,"Expected a large number"); + return NULL; + } +} + +%typemap(out) time_t +{ + $result = PyLong_FromLong((long)$1); +} + +%typemap(in) int64_t +{ + if (PyLong_Check($input)) + $1 = (int64_t) PyLong_AsLong($input); + else if (PyInt_Check($input)) + $1 = (int64_t) PyInt_AsLong($input); + else if (PyFloat_Check($input)) + $1 = (int64_t) PyFloat_AsDouble($input); + else { + PyErr_SetString(PyExc_TypeError,"Expected a large number"); + return NULL; + } +} + +%typemap(out) int64_t +{ + $result = PyLong_FromLong((long)$1); +} + +#define __LA_INT64_T long long +#define __LA_MODE_T int + +/* STRUCTURES */ +struct archive; +struct archive_entry; + +/* ARCHIVE READING */ +extern struct archive *archive_read_new(void); +extern int archive_read_free(struct archive *); + +/* opening */ +extern int archive_read_open_filename(struct archive *, + const char *_filename, size_t _block_size); +extern int archive_read_open_memory(struct archive *, + void * buff, size_t size); +extern int archive_read_open_memory2(struct archive *a, void *buff, + size_t size, size_t read_size); +extern int archive_read_open_fd(struct archive *, int _fd, + size_t _block_size); + +/* closing */ +extern int archive_read_close(struct archive *); +extern int archive_format(struct archive *); + +/* headers */ +extern int archive_read_next_header2(struct archive *, + struct archive_entry *); +extern const struct stat *archive_entry_stat(struct archive_entry *); +extern __LA_INT64_T archive_read_header_position(struct archive *); + +/* data */ +extern int archive_read_data_skip(struct archive *); +extern int archive_read_data_into_fd(struct archive *, int fd); + +/* FILTERS */ +extern int archive_read_support_filter_all(struct archive *); +extern int archive_read_support_filter_bzip2(struct archive *); +extern int archive_read_support_filter_compress(struct archive *); +extern int archive_read_support_filter_gzip(struct archive *); +extern int archive_read_support_filter_lzip(struct archive *); +extern int archive_read_support_filter_lzma(struct archive *); +extern int archive_read_support_filter_none(struct archive *); +extern int archive_read_support_filter_rpm(struct archive *); +extern int archive_read_support_filter_uu(struct archive *); +extern int archive_read_support_filter_xz(struct archive *); +extern int archive_read_support_filter_zstd(struct archive *); + +extern int archive_filter_count(struct archive *); +extern const char * archive_filter_name(struct archive *, int); + +/* FORMATS */ +extern int archive_read_support_format_all(struct archive *); +extern int archive_read_support_format_7zip(struct archive *); +extern int archive_read_support_format_ar(struct archive *); +extern int archive_read_support_format_cab(struct archive *); +extern int archive_read_support_format_cpio(struct archive *); +extern int archive_read_support_format_empty(struct archive *); +extern int archive_read_support_format_gnutar(struct archive *); +extern int archive_read_support_format_iso9660(struct archive *); +extern int archive_read_support_format_lha(struct archive *); +/*extern int archive_read_support_format_mtree(struct archive *);*/ +extern int archive_read_support_format_rar(struct archive *); +extern int archive_read_support_format_raw(struct archive *); +extern int archive_read_support_format_tar(struct archive *); +extern int archive_read_support_format_xar(struct archive *); +extern int archive_read_support_format_zip(struct archive *); +/*extern int archive_read_support_format_by_code(struct archive *, int);*/ + +/* OPTIONS */ +extern int archive_write_set_bytes_in_last_block(struct archive *_a, int bytes_in_last_block); +extern int archive_write_set_filter_option(struct archive *_a, const char *m, const char *o, const char *v); +extern int archive_write_zip_set_compression_deflate(struct archive *_a); +extern int archive_write_set_format_option(struct archive *_a, const char *m, const char *o, const char *v); +extern int archive_read_set_filter_option(struct archive *_a, const char *m, const char *o, const char *v); +extern int archive_read_set_format_option(struct archive *_a, const char *m, const char *o, const char *v); + +/* ARCHIVE WRITING */ +extern struct archive *archive_write_new(void); +extern int archive_write_free(struct archive *); + +/* opening */ +extern int archive_write_open(struct archive *, void *, + archive_open_callback *, archive_write_callback *, + archive_close_callback *); +extern int archive_write_open_fd(struct archive *, int _fd); +extern int archive_write_open_filename(struct archive *, const char *_file); +extern int archive_write_open_filename_w(struct archive *, + const wchar_t *_file); +extern int archive_write_open_memory(struct archive *, + void *_buffer, size_t _buffSize, size_t *_used); + +/* closing */ +extern int archive_write_close(struct archive *); + +/* headers */ +extern int archive_write_header(struct archive *, + struct archive_entry *); + +/* data */ + +/* commit */ +extern int archive_write_finish_entry(struct archive *); + +/* FILTERS */ +extern int archive_write_add_filter_bzip2(struct archive *); +extern int archive_write_add_filter_compress(struct archive *); +extern int archive_write_add_filter_gzip(struct archive *); +extern int archive_write_add_filter_lzip(struct archive *); +extern int archive_write_add_filter_lzma(struct archive *); +extern int archive_write_add_filter_none(struct archive *); +extern int archive_write_add_filter_xz(struct archive *); +extern int archive_write_add_filter_zstd(struct archive *); + + +/* FORMATS */ +/* A convenience function to set the format based on the code or name. */ +extern int archive_write_set_format(struct archive *, int format_code); +extern int archive_write_set_format_by_name(struct archive *, + const char *name); +/* To minimize link pollution, use one or more of the following. */ +extern int archive_write_set_format_ar_bsd(struct archive *); +extern int archive_write_set_format_ar_svr4(struct archive *); +extern int archive_write_set_format_cpio(struct archive *); +extern int archive_write_set_format_cpio_newc(struct archive *); +extern int archive_write_set_format_gnutar(struct archive *); +extern int archive_write_set_format_iso9660(struct archive *); +/*extern int archive_write_set_format_mtree(struct archive *);*/ +/* TODO: int archive_write_set_format_old_tar(struct archive *); */ +extern int archive_write_set_format_pax(struct archive *); +extern int archive_write_set_format_pax_restricted(struct archive *); +extern int archive_write_set_format_shar(struct archive *); +extern int archive_write_set_format_shar_dump(struct archive *); +extern int archive_write_set_format_ustar(struct archive *); +extern int archive_write_set_format_xar(struct archive *); +extern int archive_write_set_format_zip(struct archive *); + +/* ARCHIVE ENTRY */ +extern struct archive_entry *archive_entry_new(void); +extern void archive_entry_free(struct archive_entry *); +extern const char *archive_entry_symlink(struct archive_entry *); +extern void archive_entry_set_symlink(struct archive_entry *, const char *); +extern const char *archive_entry_hardlink(struct archive_entry *); +extern void archive_entry_set_hardlink(struct archive_entry *, const char *); + +/* ARCHIVE ENTRY PROPERTY ACCESS */ +/* reading */ +extern const char *archive_entry_pathname(struct archive_entry *); +extern const wchar_t *archive_entry_pathname_w(struct archive_entry *); +extern __LA_INT64_T archive_entry_size(struct archive_entry *); +extern time_t archive_entry_mtime(struct archive_entry *); +extern time_t archive_entry_mtime_nsec(struct archive_entry *); +extern __LA_MODE_T archive_entry_filetype(struct archive_entry *); +extern __LA_MODE_T archive_entry_perm(struct archive_entry *); + +/* writing */ +extern void archive_entry_set_pathname(struct archive_entry *, const char *); +extern void archive_entry_set_size(struct archive_entry *, __LA_INT64_T); +extern void archive_entry_set_mtime(struct archive_entry *, time_t, long); +extern void archive_entry_set_filetype(struct archive_entry *, unsigned int); +extern void archive_entry_set_perm(struct archive_entry *, __LA_MODE_T); + + +/* ERROR HANDLING */ +extern int archive_errno(struct archive *); +extern const char *archive_error_string(struct archive *); + + +/* CONSTANTS */ +#define ARCHIVE_VERSION_NUMBER 3000001 +#define ARCHIVE_VERSION_STRING "libarchive 3.0.1b" +#define ARCHIVE_EOF 1 /* Found end of archive. */ +#define ARCHIVE_OK 0 /* Operation was successful. */ +#define ARCHIVE_RETRY (-10) /* Retry might succeed. */ +#define ARCHIVE_WARN (-20) /* Partial success. */ +#define ARCHIVE_FAILED (-25) /* Current operation cannot complete. */ +#define ARCHIVE_FATAL (-30) /* No more operations are possible. */ + +#define ARCHIVE_FILTER_NONE 0 +#define ARCHIVE_FILTER_GZIP 1 +#define ARCHIVE_FILTER_BZIP2 2 +#define ARCHIVE_FILTER_COMPRESS 3 +#define ARCHIVE_FILTER_PROGRAM 4 +#define ARCHIVE_FILTER_LZMA 5 +#define ARCHIVE_FILTER_XZ 6 +#define ARCHIVE_FILTER_UU 7 +#define ARCHIVE_FILTER_RPM 8 +#define ARCHIVE_FILTER_LZIP 9 + +#define ARCHIVE_FORMAT_BASE_MASK 0xff0000 +#define ARCHIVE_FORMAT_CPIO 0x10000 +#define ARCHIVE_FORMAT_CPIO_POSIX (ARCHIVE_FORMAT_CPIO | 1) +#define ARCHIVE_FORMAT_CPIO_BIN_LE (ARCHIVE_FORMAT_CPIO | 2) +#define ARCHIVE_FORMAT_CPIO_BIN_BE (ARCHIVE_FORMAT_CPIO | 3) +#define ARCHIVE_FORMAT_CPIO_SVR4_NOCRC (ARCHIVE_FORMAT_CPIO | 4) +#define ARCHIVE_FORMAT_CPIO_SVR4_CRC (ARCHIVE_FORMAT_CPIO | 5) +#define ARCHIVE_FORMAT_CPIO_AFIO_LARGE (ARCHIVE_FORMAT_CPIO | 6) +#define ARCHIVE_FORMAT_SHAR 0x20000 +#define ARCHIVE_FORMAT_SHAR_BASE (ARCHIVE_FORMAT_SHAR | 1) +#define ARCHIVE_FORMAT_SHAR_DUMP (ARCHIVE_FORMAT_SHAR | 2) +#define ARCHIVE_FORMAT_TAR 0x30000 +#define ARCHIVE_FORMAT_TAR_USTAR (ARCHIVE_FORMAT_TAR | 1) +#define ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE (ARCHIVE_FORMAT_TAR | 2) +#define ARCHIVE_FORMAT_TAR_PAX_RESTRICTED (ARCHIVE_FORMAT_TAR | 3) +#define ARCHIVE_FORMAT_TAR_GNUTAR (ARCHIVE_FORMAT_TAR | 4) +#define ARCHIVE_FORMAT_ISO9660 0x40000 +#define ARCHIVE_FORMAT_ISO9660_ROCKRIDGE (ARCHIVE_FORMAT_ISO9660 | 1) +#define ARCHIVE_FORMAT_ZIP 0x50000 +#define ARCHIVE_FORMAT_EMPTY 0x60000 +#define ARCHIVE_FORMAT_AR 0x70000 +#define ARCHIVE_FORMAT_AR_GNU (ARCHIVE_FORMAT_AR | 1) +#define ARCHIVE_FORMAT_AR_BSD (ARCHIVE_FORMAT_AR | 2) +#define ARCHIVE_FORMAT_MTREE 0x80000 +#define ARCHIVE_FORMAT_RAW 0x90000 +#define ARCHIVE_FORMAT_XAR 0xA0000 +#define ARCHIVE_FORMAT_LHA 0xB0000 +#define ARCHIVE_FORMAT_CAB 0xC0000 +#define ARCHIVE_FORMAT_RAR 0xD0000 +#define ARCHIVE_FORMAT_7ZIP 0xE0000 + +#define ARCHIVE_EXTRACT_OWNER (0x0001) +#define ARCHIVE_EXTRACT_PERM (0x0002) +#define ARCHIVE_EXTRACT_TIME (0x0004) +#define ARCHIVE_EXTRACT_NO_OVERWRITE (0x0008) +#define ARCHIVE_EXTRACT_UNLINK (0x0010) +#define ARCHIVE_EXTRACT_ACL (0x0020) +#define ARCHIVE_EXTRACT_FFLAGS (0x0040) +#define ARCHIVE_EXTRACT_XATTR (0x0080) +#define ARCHIVE_EXTRACT_SECURE_SYMLINKS (0x0100) +#define ARCHIVE_EXTRACT_SECURE_NODOTDOT (0x0200) +#define ARCHIVE_EXTRACT_NO_AUTODIR (0x0400) +#define ARCHIVE_EXTRACT_NO_OVERWRITE_NEWER (0x0800) +#define ARCHIVE_EXTRACT_SPARSE (0x1000) +#define ARCHIVE_EXTRACT_MAC_METADATA (0x2000) + +%inline %{ +PyObject *archive_read_data_into_str(struct archive *archive, int len) { + PyObject *str = NULL; + if (!(str = PyBytes_FromStringAndSize(NULL, len))) { + PyErr_SetString(PyExc_MemoryError, "could not allocate string."); + return NULL; + } + if (len != archive_read_data(archive, PyBytes_AS_STRING(str), len)) { + PyErr_SetString(PyExc_RuntimeError, "could not read requested data."); + return NULL; + } + return str; +} + +PyObject *archive_write_data_from_str(struct archive *archive, PyObject *str) { + int len = PyBytes_Size(str); + if (len == 0) + return PyInt_FromLong(len); + int ret = archive_write_data(archive, PyBytes_AS_STRING(str), len); + if (ret == ARCHIVE_FATAL) { + PyErr_Format(PyExc_RuntimeError, "Could not write requested data - most likely no space left on device (error code: %d)", ret); + return NULL; + } + else if (ret <= 0) { + PyErr_Format(PyExc_RuntimeError, "Could not write requested data (error code: %d)", ret); + return NULL; + } + return PyInt_FromLong(len); +} +%} diff --git a/contrib/python/python-libarchive/py3/libarchive/tar.py b/contrib/python/python-libarchive/py3/libarchive/tar.py new file mode 100644 index 00000000000..f14149804b9 --- /dev/null +++ b/contrib/python/python-libarchive/py3/libarchive/tar.py @@ -0,0 +1,135 @@ +# Copyright (c) 2011, SmartFile <[email protected]> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the organization nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import time +from libarchive import is_archive, Entry, SeekableArchive +from tarfile import DEFAULT_FORMAT, USTAR_FORMAT, GNU_FORMAT, PAX_FORMAT, ENCODING +from tarfile import REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE + +FORMAT_CONVERSION = { + USTAR_FORMAT: 'tar', + GNU_FORMAT: 'gnu', + PAX_FORMAT: 'pax', +} + + +def is_tarfile(filename): + return is_archive(filename, formats=('tar', 'gnu', 'pax')) + + +def open(**kwargs): + return TarFile(**kwargs) + + +class TarInfo(Entry): + def __init__(self, name): + super(TarInfo, self).__init__(pathname=name) + + fromtarfile = Entry.from_archive + + def get_name(self): + return self.pathname + + def set_name(self, value): + self.pathname = value + + name = property(get_name, set_name) + + @property + def get_type(self): + for attr, type in ( + ('isdir', DIRTYPE), ('isfile', REGTYPE), ('issym', SYMTYPE), + ('isfifo', FIFOTYPE), ('ischr', CHRTYPE), ('isblk', BLKTYPE), + ): + if getattr(self, attr)(): + return type + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + pax_headers = property(_get_missing, _set_missing) + + +class TarFile(SeekableArchive): + def __init__(self, name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, encoding=ENCODING): + if name: + f = name + elif fileobj: + f = fileobj + try: + format = FORMAT_CONVERSON.get(format) + except KeyError: + raise Exception('Invalid tar format: %s' % format) + super(TarFile, self).__init__(f, mode=mode, format=format, entry_class=tarinfo, encoding=encoding) + + getmember = SeekableArchive.getentry + list = SeekableArchive.printlist + extract = SeekableArchive.readpath + extractfile = SeekableArchive.readstream + + def getmembers(self): + return list(self) + + def getnames(self): + return list(self.iterpaths) + + def next(self): + pass # TODO: how to do this? + + def extract(self, member, path=None): + if path is None: + path = os.getcwd() + if isinstance(member, basestring): + f = os.path.join(path, member) + else: + f = os.path.join(path, member.pathname) + return self.readpath(member, f) + + def add(self, name, arcname, recursive=True, exclude=None, filter=None): + pass # TODO: implement this. + + def addfile(tarinfo, fileobj): + return self.writepath(fileobj, tarinfo) + + def gettarinfo(name=None, arcname=None, fileobj=None): + if name: + f = name + elif fileobj: + f = fileobj + entry = self.entry_class.from_file(f) + if arcname: + entry.pathname = arcname + return entry + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + pax_headers = property(_get_missing, _set_missing) diff --git a/contrib/python/python-libarchive/py3/libarchive/zip.py b/contrib/python/python-libarchive/py3/libarchive/zip.py new file mode 100644 index 00000000000..539f6dbcc49 --- /dev/null +++ b/contrib/python/python-libarchive/py3/libarchive/zip.py @@ -0,0 +1,151 @@ +# Copyright (c) 2011, SmartFile <[email protected]> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the organization nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os, time +from libarchive import is_archive, Entry, SeekableArchive +from zipfile import ZIP_STORED, ZIP_DEFLATED + + +def is_zipfile(filename): + return is_archive(filename, formats=('zip', )) + + +class ZipEntry(Entry): + def __init__(self, *args, **kwargs): + super(ZipEntry, self).__init__(*args, **kwargs) + + def get_filename(self): + return self.pathname + + def set_filename(self, value): + self.pathname = value + + filename = property(get_filename, set_filename) + + def get_file_size(self): + return self.size + + def set_file_size(self, value): + assert isinstance(size, (int, long)), 'Please provide size as int or long.' + self.size = value + + file_size = property(get_file_size, set_file_size) + + def get_date_time(self): + return time.localtime(self.mtime)[0:6] + + def set_date_time(self, value): + assert isinstance(value, tuple), 'mtime should be tuple (year, month, day, hour, minute, second).' + assert len(value) == 6, 'mtime should be tuple (year, month, day, hour, minute, second).' + self.mtime = time.mktime(value + (0, 0, 0)) + + date_time = property(get_date_time, set_date_time) + + header_offset = Entry.header_position + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + compress_type = property(_get_missing, _set_missing) + comment = property(_get_missing, _set_missing) + extra = property(_get_missing, _set_missing) + create_system = property(_get_missing, _set_missing) + create_version = property(_get_missing, _set_missing) + extract_version = property(_get_missing, _set_missing) + reserved = property(_get_missing, _set_missing) + flag_bits = property(_get_missing, _set_missing) + volume = property(_get_missing, _set_missing) + internal_attr = property(_get_missing, _set_missing) + external_attr = property(_get_missing, _set_missing) + CRC = property(_get_missing, _set_missing) + compress_size = property(_get_missing, _set_missing) + + +class ZipFile(SeekableArchive): + def __init__(self, f, mode='r', compression=ZIP_DEFLATED, allowZip64=False): + super(ZipFile, self).__init__(f, mode=mode, format='zip', entry_class=ZipEntry, encoding='CP437') + if mode == 'w' and compression == ZIP_STORED: + # Disable compression for writing. + _libarchive.archive_write_set_format_option(self.archive._a, "zip", "compression", "store") + self.compression = compression + + getinfo = SeekableArchive.getentry + + def namelist(self): + return list(self.iterpaths) + + def infolist(self): + return list(self) + + def open(self, name, mode, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + if mode == 'r': + return self.readstream(name) + else: + return self.writestream(name) + + def extract(self, name, path=None, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + if not path: + path = os.getcwd() + return self.readpath(name, os.path.join(path, name)) + + def extractall(self, path, names=None, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + if not names: + names = self.namelist() + if names: + for name in names: + self.extract(name, path) + + def read(self, name, pwd=None): + if pwd: + raise NotImplemented('Encryption not supported.') + return self.read(name) + + def writestr(self, member, data, compress_type=None): + if compress_type != self.compression: + raise Exception('Cannot change compression type for individual entries.') + return self.write(member, data) + + def setpassword(self, pwd): + raise NotImplemented('Encryption not supported.') + + def testzip(self): + raise NotImplemented() + + def _get_missing(self): + raise NotImplemented() + + def _set_missing(self, value): + raise NotImplemented() + + comment = property(_get_missing, _set_missing) diff --git a/contrib/python/python-libarchive/py3/ya.make b/contrib/python/python-libarchive/py3/ya.make new file mode 100644 index 00000000000..a905f47a122 --- /dev/null +++ b/contrib/python/python-libarchive/py3/ya.make @@ -0,0 +1,28 @@ +PY3_LIBRARY() + +LICENSE(BSD-3-Clause) + +VERSION(3.1.2.post1) + +PEERDIR( + contrib/libs/libarchive + contrib/python/contextlib2 + contrib/python/six +) + +ADDINCL( + contrib/libs/libarchive/libarchive +) + +NO_LINT() + +PY_SRCS( + SWIG_C + TOP_LEVEL + libarchive/__init__.py + libarchive/tar.py + libarchive/zip.py + libarchive/_libarchive.swg +) + +END() diff --git a/contrib/python/python-libarchive/ya.make b/contrib/python/python-libarchive/ya.make new file mode 100644 index 00000000000..0c18f7d44ba --- /dev/null +++ b/contrib/python/python-libarchive/ya.make @@ -0,0 +1,20 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +VERSION(Service-proxy-version) + +IF (PYTHON2) + PEERDIR(contrib/python/python-libarchive/py2) +ELSE() + PEERDIR(contrib/python/python-libarchive/py3) +ENDIF() + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +) |