diff options
| author | YDBot <[email protected]> | 2026-07-13 03:53:22 +0000 |
|---|---|---|
| committer | YDBot <[email protected]> | 2026-07-13 03:53:22 +0000 |
| commit | c1fc37566f6a3639b67e7fc2e23eff1647893bd1 (patch) | |
| tree | 189dcfc42f12b41b5119111d1452727a956a915e | |
| parent | 778f44e34127879022a1e961ef7ca35d247981ab (diff) | |
| parent | a9010d99cb515f79a521fe7d26fb0c814936d972 (diff) | |
Merge pull request #46281 from ydb-platform/merge-rightlib-260713-0103
50 files changed, 956 insertions, 167 deletions
diff --git a/build/config/tests/flake8/flake8.conf b/build/config/tests/flake8/flake8.conf index a4a60be7432..4977577208e 100644 --- a/build/config/tests/flake8/flake8.conf +++ b/build/config/tests/flake8/flake8.conf @@ -9,7 +9,7 @@ select = #B, S, # bandit #C, # commas #D, # docstrings - #P, # string-format + #FMT, # string-format #Q, # quotes ignore = @@ -30,6 +30,15 @@ ignore = # line break after binary operator W504, + # format string contains unindexed parameters + FMT101, + # docstring contains unindexed parameters + FMT102, + # other string contains unindexed parameters + FMT103, + # format call uses implicit and explicit indexes together + FMT205, + # astroid-error PLF002, # function-redefined diff --git a/build/external_resources/flake8_py3/ya.make b/build/external_resources/flake8_py3/ya.make index 79961a5181a..2d50f6e09e9 100644 --- a/build/external_resources/flake8_py3/ya.make +++ b/build/external_resources/flake8_py3/ya.make @@ -12,12 +12,12 @@ ENDIF() DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( FLAKE8_PY3 - sbr:10993120939 FOR DARWIN-ARM64 - sbr:10993122025 FOR DARWIN + sbr:12763243518 FOR DARWIN-ARM64 + sbr:12763245515 FOR DARWIN sbr:6726869647 FOR LINUX-PPC64LE - sbr:10993123638 FOR LINUX - sbr:10993119916 FOR LINUX-AARCH64 - sbr:10993122759 FOR WIN32 + sbr:12763248856 FOR LINUX + sbr:12763242238 FOR LINUX-AARCH64 + sbr:12763247016 FOR WIN32 ) END() diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 3a396ebb3e8..4f6ac9c54fd 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -474,6 +474,7 @@ "6812259929": "{registry_endpoint}/6812259929", "10531322967": "{registry_endpoint}/10531322967", "10993122025": "{registry_endpoint}/10993122025", + "12763245515": "{registry_endpoint}/12763245515", "6603784347": "{registry_endpoint}/6603784347", "6652224583": "{registry_endpoint}/6652224583", "6674734480": "{registry_endpoint}/6674734480", @@ -485,6 +486,7 @@ "9245124509": "{registry_endpoint}/9245124509", "10531314816": "{registry_endpoint}/10531314816", "10993120939": "{registry_endpoint}/10993120939", + "12763243518": "{registry_endpoint}/12763243518", "6603781997": "{registry_endpoint}/6603781997", "6652223452": "{registry_endpoint}/6652223452", "6674733966": "{registry_endpoint}/6674733966", @@ -496,6 +498,7 @@ "9245123262": "{registry_endpoint}/9245123262", "10531337045": "{registry_endpoint}/10531337045", "10993123638": "{registry_endpoint}/10993123638", + "12763248856": "{registry_endpoint}/12763248856", "6603787904": "{registry_endpoint}/6603787904", "6652225714": "{registry_endpoint}/6652225714", "6674734993": "{registry_endpoint}/6674734993", @@ -507,6 +510,7 @@ "9245126434": "{registry_endpoint}/9245126434", "10531306931": "{registry_endpoint}/10531306931", "10993119916": "{registry_endpoint}/10993119916", + "12763242238": "{registry_endpoint}/12763242238", "6603780916": "{registry_endpoint}/6603780916", "6652222817": "{registry_endpoint}/6652222817", "6674733653": "{registry_endpoint}/6674733653", @@ -522,6 +526,7 @@ "6726869647": "{registry_endpoint}/6726869647", "10531330161": "{registry_endpoint}/10531330161", "10993122759": "{registry_endpoint}/10993122759", + "12763247016": "{registry_endpoint}/12763247016", "6603785992": "{registry_endpoint}/6603785992", "6652225128": "{registry_endpoint}/6652225128", "6674734716": "{registry_endpoint}/6674734716", @@ -2879,6 +2884,7 @@ "6812259929": "devtools/local_cache/toolscache/server/ya-tc for linux-aarch64", "10531322967": "devtools/ya/test/programs/flake8/flake8 for darwin", "10993122025": "devtools/ya/test/programs/flake8/flake8 for darwin", + "12763245515": "devtools/ya/test/programs/flake8/flake8 for darwin", "6603784347": "devtools/ya/test/programs/flake8/flake8 for darwin", "6652224583": "devtools/ya/test/programs/flake8/flake8 for darwin", "6674734480": "devtools/ya/test/programs/flake8/flake8 for darwin", @@ -2890,6 +2896,7 @@ "9245124509": "devtools/ya/test/programs/flake8/flake8 for darwin", "10531314816": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "10993120939": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", + "12763243518": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6603781997": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6652223452": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6674733966": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", @@ -2901,6 +2908,7 @@ "9245123262": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "10531337045": "devtools/ya/test/programs/flake8/flake8 for linux", "10993123638": "devtools/ya/test/programs/flake8/flake8 for linux", + "12763248856": "devtools/ya/test/programs/flake8/flake8 for linux", "6603787904": "devtools/ya/test/programs/flake8/flake8 for linux", "6652225714": "devtools/ya/test/programs/flake8/flake8 for linux", "6674734993": "devtools/ya/test/programs/flake8/flake8 for linux", @@ -2912,6 +2920,7 @@ "9245126434": "devtools/ya/test/programs/flake8/flake8 for linux", "10531306931": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "10993119916": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", + "12763242238": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6603780916": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6652222817": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6674733653": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", @@ -2927,6 +2936,7 @@ "6726869647": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le", "10531330161": "devtools/ya/test/programs/flake8/flake8 for win32", "10993122759": "devtools/ya/test/programs/flake8/flake8 for win32", + "12763247016": "devtools/ya/test/programs/flake8/flake8 for win32", "6603785992": "devtools/ya/test/programs/flake8/flake8 for win32", "6652225128": "devtools/ya/test/programs/flake8/flake8 for win32", "6674734716": "devtools/ya/test/programs/flake8/flake8 for win32", diff --git a/contrib/python/s3transfer/py3/.dist-info/METADATA b/contrib/python/s3transfer/py3/.dist-info/METADATA index 629c988c78a..cd598e2d609 100644 --- a/contrib/python/s3transfer/py3/.dist-info/METADATA +++ b/contrib/python/s3transfer/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: s3transfer -Version: 0.11.0 +Version: 0.11.3 Summary: An Amazon S3 Transfer Manager Home-page: https://github.com/boto/s3transfer Author: Amazon Web Services @@ -22,9 +22,9 @@ Classifier: Programming Language :: Python :: 3.13 Requires-Python: >= 3.8 License-File: LICENSE.txt License-File: NOTICE.txt -Requires-Dist: botocore (<2.0a.0,>=1.33.2) +Requires-Dist: botocore (<2.0a.0,>=1.36.0) Provides-Extra: crt -Requires-Dist: botocore[crt] (<2.0a.0,>=1.33.2) ; extra == 'crt' +Requires-Dist: botocore[crt] (<2.0a.0,>=1.36.0) ; extra == 'crt' ===================================================== s3transfer - An Amazon S3 Transfer Manager for Python diff --git a/contrib/python/s3transfer/py3/s3transfer/__init__.py b/contrib/python/s3transfer/py3/s3transfer/__init__.py index de62a74eba5..a15bac0cb9f 100644 --- a/contrib/python/s3transfer/py3/s3transfer/__init__.py +++ b/contrib/python/s3transfer/py3/s3transfer/__init__.py @@ -145,7 +145,7 @@ import s3transfer.compat from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError __author__ = 'Amazon Web Services' -__version__ = '0.11.0' +__version__ = '0.11.3' class NullHandler(logging.Handler): diff --git a/contrib/python/s3transfer/py3/s3transfer/crt.py b/contrib/python/s3transfer/py3/s3transfer/crt.py index 37b041d8a96..b11c368261c 100644 --- a/contrib/python/s3transfer/py3/s3transfer/crt.py +++ b/contrib/python/s3transfer/py3/s3transfer/crt.py @@ -895,14 +895,22 @@ class S3ClientArgsCreator: ) and accesspoint_arn_details['region'] == "": # Configure our region to `*` to propogate in `x-amz-region-set` # for multi-region support in MRAP accesspoints. + # use_double_uri_encode and should_normalize_uri_path are defaulted to be True + # But SDK already encoded the URI, and it's for S3, so set both to False make_request_args['signing_config'] = AwsSigningConfig( algorithm=AwsSigningAlgorithm.V4_ASYMMETRIC, region="*", + use_double_uri_encode=False, + should_normalize_uri_path=False, ) call_args.bucket = accesspoint_arn_details['resource_name'] elif is_s3express_bucket(call_args.bucket): + # use_double_uri_encode and should_normalize_uri_path are defaulted to be True + # But SDK already encoded the URI, and it's for S3, so set both to False make_request_args['signing_config'] = AwsSigningConfig( - algorithm=AwsSigningAlgorithm.V4_S3EXPRESS + algorithm=AwsSigningAlgorithm.V4_S3EXPRESS, + use_double_uri_encode=False, + should_normalize_uri_path=False, ) return make_request_args diff --git a/contrib/python/s3transfer/py3/s3transfer/manager.py b/contrib/python/s3transfer/py3/s3transfer/manager.py index 23b6f305ef0..8e1cdf068bb 100644 --- a/contrib/python/s3transfer/py3/s3transfer/manager.py +++ b/contrib/python/s3transfer/py3/s3transfer/manager.py @@ -519,7 +519,11 @@ class TransferManager: ) def _add_operation_defaults(self, extra_args): - set_default_checksum_algorithm(extra_args) + if ( + self.client.meta.config.request_checksum_calculation + == "when_supported" + ): + set_default_checksum_algorithm(extra_args) def _submit_transfer( self, call_args, submission_task_cls, extra_main_kwargs=None diff --git a/contrib/python/s3transfer/py3/tests/functional/test_crt.py b/contrib/python/s3transfer/py3/tests/functional/test_crt.py index 5ef25144e85..98e3fffbe7a 100644 --- a/contrib/python/s3transfer/py3/tests/functional/test_crt.py +++ b/contrib/python/s3transfer/py3/tests/functional/test_crt.py @@ -163,6 +163,12 @@ class TestCRTTransferManager(unittest.TestCase): make_request_kwargs['signing_config'].algorithm, awscrt.auth.AwsSigningAlgorithm.V4_S3EXPRESS, ) + self.assertFalse( + make_request_kwargs['signing_config'].use_double_uri_encode, + ) + self.assertFalse( + make_request_kwargs['signing_config'].should_normalize_uri_path, + ) def _assert_expected_mrap_request( self, make_request_kwargs, expected_http_method='GET' @@ -179,6 +185,12 @@ class TestCRTTransferManager(unittest.TestCase): awscrt.auth.AwsSigningAlgorithm.V4_ASYMMETRIC, ) self.assertEqual(make_request_kwargs['signing_config'].region, "*") + self.assertFalse( + make_request_kwargs['signing_config'].use_double_uri_encode, + ) + self.assertFalse( + make_request_kwargs['signing_config'].should_normalize_uri_path, + ) def _assert_subscribers_called(self, expected_future=None): self.assertTrue(self.record_subscriber.on_queued_called) diff --git a/contrib/python/s3transfer/py3/tests/functional/test_upload.py b/contrib/python/s3transfer/py3/tests/functional/test_upload.py index 00ac284f62f..efbc55af06c 100644 --- a/contrib/python/s3transfer/py3/tests/functional/test_upload.py +++ b/contrib/python/s3transfer/py3/tests/functional/test_upload.py @@ -19,6 +19,7 @@ from io import BytesIO from botocore.awsrequest import AWSRequest from botocore.client import Config from botocore.exceptions import ClientError +from botocore.httpchecksum import DEFAULT_CHECKSUM_ALGORITHM from botocore.stub import ANY from s3transfer.manager import TransferConfig, TransferManager @@ -143,7 +144,7 @@ class TestNonMultipartUpload(BaseUploadTest): __test__ = True def add_put_object_response_with_default_expected_params( - self, extra_expected_params=None, bucket=None + self, extra_expected_params=None, bucket=None, include_checksum=True ): if bucket is None: bucket = self.bucket @@ -152,8 +153,9 @@ class TestNonMultipartUpload(BaseUploadTest): 'Body': ANY, 'Bucket': bucket, 'Key': self.key, - 'ChecksumAlgorithm': 'CRC32', } + if include_checksum: + expected_params["ChecksumAlgorithm"] = DEFAULT_CHECKSUM_ALGORITHM if extra_expected_params: expected_params.update(extra_expected_params) upload_response = self.create_stubbed_responses()[0] @@ -187,6 +189,46 @@ class TestNonMultipartUpload(BaseUploadTest): self.assert_expected_client_calls_were_correct() self.assert_put_object_body_was_correct() + def test_upload_with_default_checksum_when_supported(self): + # Reset client to configure `request_checksum_calculation` to "when_supported". + self.reset_stubber_with_new_client( + {'config': Config(request_checksum_calculation="when_supported")} + ) + self.client.meta.events.register( + 'before-parameter-build.s3.*', self.collect_body + ) + self._manager = TransferManager(self.client, self.config) + + self.add_put_object_response_with_default_expected_params( + include_checksum=True + ) + future = self.manager.upload( + self.filename, self.bucket, self.key, self.extra_args + ) + future.result() + self.assert_expected_client_calls_were_correct() + self.assert_put_object_body_was_correct() + + def test_upload_with_default_checksum_when_required(self): + # Reset client to configure `request_checksum_calculation` to "when_required". + self.reset_stubber_with_new_client( + {'config': Config(request_checksum_calculation="when_required")} + ) + self.client.meta.events.register( + 'before-parameter-build.s3.*', self.collect_body + ) + self._manager = TransferManager(self.client, self.config) + + self.add_put_object_response_with_default_expected_params( + include_checksum=False + ) + future = self.manager.upload( + self.filename, self.bucket, self.key, self.extra_args + ) + future.result() + self.assert_expected_client_calls_were_correct() + self.assert_put_object_body_was_correct() + def test_upload_with_s3express_default_checksum(self): s3express_bucket = "mytestbucket--usw2-az6--x-s3" self.assertFalse("ChecksumAlgorithm" in self.extra_args) @@ -370,9 +412,7 @@ class TestMultipartUpload(BaseUploadTest): self.assertEqual(self.sent_bodies, expected_contents) def add_create_multipart_response_with_default_expected_params( - self, - extra_expected_params=None, - bucket=None, + self, extra_expected_params=None, bucket=None, include_checksum=True ): if bucket is None: bucket = self.bucket @@ -380,8 +420,9 @@ class TestMultipartUpload(BaseUploadTest): expected_params = { 'Bucket': bucket, 'Key': self.key, - 'ChecksumAlgorithm': 'CRC32', } + if include_checksum: + expected_params["ChecksumAlgorithm"] = DEFAULT_CHECKSUM_ALGORITHM if extra_expected_params: expected_params.update(extra_expected_params) response = self.create_stubbed_responses()[0] @@ -389,9 +430,7 @@ class TestMultipartUpload(BaseUploadTest): self.stubber.add_response(**response) def add_upload_part_responses_with_default_expected_params( - self, - extra_expected_params=None, - bucket=None, + self, extra_expected_params=None, bucket=None, include_checksum=True ): if bucket is None: bucket = self.bucket @@ -406,50 +445,48 @@ class TestMultipartUpload(BaseUploadTest): 'UploadId': self.multipart_id, 'Body': ANY, 'PartNumber': i + 1, - 'ChecksumAlgorithm': 'CRC32', } + if include_checksum: + expected_params["ChecksumAlgorithm"] = ( + DEFAULT_CHECKSUM_ALGORITHM + ) if extra_expected_params: expected_params.update(extra_expected_params) - name = expected_params['ChecksumAlgorithm'] - checksum_member = f'Checksum{name.upper()}' - response = upload_part_response['service_response'] - response[checksum_member] = f'sum{i+1}==' + # If ChecksumAlgorithm is in expected parameters, add checksum to the response + checksum_algorithm = expected_params.get('ChecksumAlgorithm') + if checksum_algorithm: + checksum_member = f'Checksum{checksum_algorithm.upper()}' + response = upload_part_response['service_response'] + response[checksum_member] = f'sum{i+1}==' upload_part_response['expected_params'] = expected_params self.stubber.add_response(**upload_part_response) def add_complete_multipart_response_with_default_expected_params( - self, - extra_expected_params=None, - bucket=None, + self, extra_expected_params=None, bucket=None, include_checksum=True ): if bucket is None: bucket = self.bucket + num_parts = 3 + parts = [] + for part_num in range(1, num_parts + 1): + part = { + 'ETag': f'etag-{part_num}', + 'PartNumber': part_num, + } + if include_checksum: + part[f"Checksum{DEFAULT_CHECKSUM_ALGORITHM}"] = ( + f"sum{part_num}==" + ) + parts.append(part) + expected_params = { 'Bucket': bucket, 'Key': self.key, 'UploadId': self.multipart_id, - 'MultipartUpload': { - 'Parts': [ - { - 'ETag': 'etag-1', - 'PartNumber': 1, - 'ChecksumCRC32': 'sum1==', - }, - { - 'ETag': 'etag-2', - 'PartNumber': 2, - 'ChecksumCRC32': 'sum2==', - }, - { - 'ETag': 'etag-3', - 'PartNumber': 3, - 'ChecksumCRC32': 'sum3==', - }, - ] - }, + 'MultipartUpload': {'Parts': parts}, } if extra_expected_params: expected_params.update(extra_expected_params) @@ -761,3 +798,53 @@ class TestMultipartUpload(BaseUploadTest): ) future.result() self.assert_expected_client_calls_were_correct() + + def test_multipart_upload_with_default_checksum_when_supported(self): + # Reset client to configure `request_checksum_calculation` to "when_supported". + self.reset_stubber_with_new_client( + {'config': Config(request_checksum_calculation="when_supported")} + ) + self.client.meta.events.register( + 'before-parameter-build.s3.*', self.collect_body + ) + self._manager = TransferManager(self.client, self.config) + + self.add_create_multipart_response_with_default_expected_params( + include_checksum=True + ) + self.add_upload_part_responses_with_default_expected_params( + include_checksum=True + ) + self.add_complete_multipart_response_with_default_expected_params() + + future = self.manager.upload( + self.filename, self.bucket, self.key, self.extra_args + ) + future.result() + self.assert_expected_client_calls_were_correct() + + def test_multipart_upload_with_default_checksum_when_required(self): + # Reset client to configure `request_checksum_calculation` to "when_required". + self.reset_stubber_with_new_client( + {'config': Config(request_checksum_calculation="when_required")} + ) + self.client.meta.events.register( + 'before-parameter-build.s3.*', self.collect_body + ) + self._manager = TransferManager(self.client, self.config) + + self.add_create_multipart_response_with_default_expected_params( + include_checksum=False + ) + self.add_upload_part_responses_with_default_expected_params( + include_checksum=False + ) + self.add_complete_multipart_response_with_default_expected_params( + include_checksum=False + ) + + future = self.manager.upload( + self.filename, self.bucket, self.key, self.extra_args + ) + future.result() + self.assert_expected_client_calls_were_correct() diff --git a/contrib/python/s3transfer/py3/ya.make b/contrib/python/s3transfer/py3/ya.make index 57776c6f7ce..7c6a4458dcf 100644 --- a/contrib/python/s3transfer/py3/ya.make +++ b/contrib/python/s3transfer/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(0.11.0) +VERSION(0.11.3) LICENSE(Apache-2.0) diff --git a/contrib/python/scramp/.dist-info/METADATA b/contrib/python/scramp/.dist-info/METADATA index d3b2cda4342..eb6b3e790f8 100644 --- a/contrib/python/scramp/.dist-info/METADATA +++ b/contrib/python/scramp/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: scramp -Version: 1.4.9 +Version: 1.4.10 Summary: An implementation of the SCRAM protocol. Project-URL: Homepage, https://codeberg.org/tlocke/scramp Author: The Contributors @@ -54,6 +54,7 @@ Scramp supports the following mechanisms: * [OpenSSF Scorecard](#openssf-scorecard) * [Doing A Release Of Scramp](#doing-a-release-of-scramp) * [Release Notes](#release-notes) + * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27) * [Version 1.4.9, 2026-06-19](#version-149-2026-06-19) * [Version 1.4.8, 2026-01-06](#version-148-2026-01-06) * [Version 1.4.7, 2026-01-04](#version-147-2026-01-04) @@ -384,6 +385,15 @@ Run `tox` to make sure all tests pass, then update the release notes, then do: ## Release Notes +### Version 1.4.10, 2026-06-27 + +- Implement max and min guards for iteration count. +- The xor() function should fail if the two values are of unequal lengths +- Handle the case of an error in first server message +- Fix formatting in channel binding error message +- SHA3-512 didn't work, but it should do now + + ### Version 1.4.9, 2026-06-19 - Fix typo in error message. diff --git a/contrib/python/scramp/README.md b/contrib/python/scramp/README.md index 1ccdd745c56..652f8f590d0 100644 --- a/contrib/python/scramp/README.md +++ b/contrib/python/scramp/README.md @@ -33,6 +33,7 @@ Scramp supports the following mechanisms: * [OpenSSF Scorecard](#openssf-scorecard) * [Doing A Release Of Scramp](#doing-a-release-of-scramp) * [Release Notes](#release-notes) + * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27) * [Version 1.4.9, 2026-06-19](#version-149-2026-06-19) * [Version 1.4.8, 2026-01-06](#version-148-2026-01-06) * [Version 1.4.7, 2026-01-04](#version-147-2026-01-04) @@ -363,6 +364,15 @@ Run `tox` to make sure all tests pass, then update the release notes, then do: ## Release Notes +### Version 1.4.10, 2026-06-27 + +- Implement max and min guards for iteration count. +- The xor() function should fail if the two values are of unequal lengths +- Handle the case of an error in first server message +- Fix formatting in channel binding error message +- SHA3-512 didn't work, but it should do now + + ### Version 1.4.9, 2026-06-19 - Fix typo in error message. diff --git a/contrib/python/scramp/scramp/core.py b/contrib/python/scramp/scramp/core.py index 7bfd362914a..7ab1a5703a7 100644 --- a/contrib/python/scramp/scramp/core.py +++ b/contrib/python/scramp/scramp/core.py @@ -77,6 +77,8 @@ CHANNEL_TYPES = ( "tls-unique-for-telnet", ) +MAX_ITERATION_COUNT = 10_000_000 # DoS guard + def _make_cb_data(name, ssl_socket): if name == "tls-unique": @@ -137,6 +139,10 @@ class ScramMechanism: def make_auth_info(self, password, iteration_count=None, salt=None): if iteration_count is None: iteration_count = self.iteration_count + if iteration_count < self.iteration_count: + raise ScramException( + f"The iteration count must be at least {self.iteration_count}" + ) salt, stored_key, server_key = _make_auth_info( self.hf, password, iteration_count, salt=salt ) @@ -175,8 +181,8 @@ def _validate_channel_binding(channel_binding): channel_type, channel_data = channel_binding if channel_type not in CHANNEL_TYPES: raise ScramException( - "The channel_binding parameter must either be None or a tuple with the " - "first element a str specifying one of the channel types {CHANNEL_TYPES}." + f"The channel_binding parameter must either be None or a tuple with the " + f"first element a str specifying one of the channel types {CHANNEL_TYPES}." ) if not isinstance(channel_data, bytes): @@ -207,6 +213,7 @@ class ScramClient: mech = sorted(mechs, key=attrgetter("strength"))[-1] self.hf, self.use_binding = mech.hf, mech.use_binding self.mechanism_name = mech.name + self.iterations = mech.iteration_count self.c_nonce = _make_nonce() if c_nonce is None else c_nonce self.username = username @@ -229,7 +236,7 @@ class ScramClient: self._set_stage(ClientStage.set_server_first) self.server_first = message self.nonce, self.salt, self.iterations = _set_server_first( - message, self.c_nonce + message, self.c_nonce, self.iterations ) def get_client_final(self): @@ -351,8 +358,7 @@ def _make_auth_message(client_first_bare, server_first, client_final_without_pro def _make_salted_password(hf, password, salt, iterations): - hash_name = hf.__name__.split("_")[-1] - return hashlib.pbkdf2_hmac(hash_name, uenc(saslprep(password)), salt, iterations) + return hashlib.pbkdf2_hmac(hf().name, uenc(saslprep(password)), salt, iterations) def _c_key_stored_key_s_key(hf, salted_password): @@ -518,14 +524,19 @@ def _get_server_first(nonce, salt, iterations): return ",".join((f"r={nonce}", f"s={salt}", f"i={iterations}")) -def _set_server_first(server_first, c_nonce): - msg = _parse_message(server_first, "server first", {"r", "s", "i"}) +def _set_server_first(server_first, c_nonce, min_iteration_count): + msg = _parse_message(server_first, "server first", {"r", "s", "i"}, {"e"}) if "e" in msg: raise ScramException(f"The server returned the error: {msg['e']}") nonce = msg["r"] salt = msg["s"] iterations = int(msg["i"]) + if not (min_iteration_count <= iterations <= MAX_ITERATION_COUNT): + raise ScramException( + f"Server iteration count must be between {min_iteration_count} " + f"and {MAX_ITERATION_COUNT} inclusive" + ) if not nonce.startswith(c_nonce): raise ScramException("Client nonce doesn't match.", SERVER_ERROR_OTHER_ERROR) diff --git a/contrib/python/scramp/scramp/utils.py b/contrib/python/scramp/scramp/utils.py index da864d2fe2f..da991644090 100644 --- a/contrib/python/scramp/scramp/utils.py +++ b/contrib/python/scramp/scramp/utils.py @@ -14,7 +14,7 @@ def h(hf, msg): def xor(bytes1, bytes2): - return bytes(a ^ b for a, b in zip(bytes1, bytes2)) + return bytes(a ^ b for a, b in zip(bytes1, bytes2, strict=True)) def b64enc(binary): diff --git a/contrib/python/scramp/ya.make b/contrib/python/scramp/ya.make index 9453f63f0af..f5d6699f70a 100644 --- a/contrib/python/scramp/ya.make +++ b/contrib/python/scramp/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.4.9) +VERSION(1.4.10) LICENSE(MIT-0) diff --git a/library/cpp/yt/coding/bit_io-inl.h b/library/cpp/yt/coding/bit_io-inl.h new file mode 100644 index 00000000000..89d69be6251 --- /dev/null +++ b/library/cpp/yt/coding/bit_io-inl.h @@ -0,0 +1,69 @@ +#ifndef BIT_IO_INL_H_ +#error "Direct inclusion of this file is not allowed, include bit_io.h" +// For the sake of sane code completion. +#include "bit_io.h" +#endif + +#include <util/system/compiler.h> +#include <util/system/unaligned_mem.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +inline TBitWriter::TBitWriter(char* ptr) + : Ptr_(ptr) +{ } + +Y_FORCE_INLINE void TBitWriter::WriteBits(ui32 value, int width) +{ + // Flush a whole 32-bit word at once (a single unaligned store) instead of + // looping over individual bytes with a data-dependent trip count. + Accumulator_ = (Accumulator_ << width) | value; + BitCount_ += width; + if (BitCount_ >= 32) { + BitCount_ -= 32; + ui32 word = __builtin_bswap32(static_cast<ui32>(Accumulator_ >> BitCount_)); + WriteUnaligned<ui32>(Ptr_, word); + Ptr_ += sizeof(word); + } +} + +inline char* TBitWriter::Finish() +{ + while (BitCount_ >= 8) { + BitCount_ -= 8; + *Ptr_++ = static_cast<char>((Accumulator_ >> BitCount_) & 0xff); + } + if (BitCount_ > 0) { + *Ptr_++ = static_cast<char>((Accumulator_ << (8 - BitCount_)) & 0xff); + BitCount_ = 0; + } + Accumulator_ = 0; + return Ptr_; +} + +//////////////////////////////////////////////////////////////////////////////// + +inline TBitReader::TBitReader(const char* ptr) + : Ptr_(reinterpret_cast<const ui8*>(ptr)) +{ } + +Y_FORCE_INLINE ui32 TBitReader::ReadBits(int width) +{ + while (BitCount_ < width) { + Accumulator_ = (Accumulator_ << 8) | *Ptr_++; + BitCount_ += 8; + } + BitCount_ -= width; + return static_cast<ui32>((Accumulator_ >> BitCount_) & ((1ull << width) - 1)); +} + +inline const char* TBitReader::Finish() +{ + return reinterpret_cast<const char*>(Ptr_); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/coding/bit_io.h b/library/cpp/yt/coding/bit_io.h new file mode 100644 index 00000000000..18900d69f85 --- /dev/null +++ b/library/cpp/yt/coding/bit_io.h @@ -0,0 +1,63 @@ +#pragma once + +#include <util/system/types.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +//! MSB-first bit writer over a caller-owned buffer. +/*! + * Bits are flushed to the buffer in 4-byte chunks, so the buffer must have room + * for up to 3 bytes beyond the last logically-written byte before #Finish is + * called. + */ +class TBitWriter +{ +public: + explicit TBitWriter(char* ptr); + + //! Appends the #width low bits of #value. Requires 0 <= #width <= 32 and + //! #value < 2^#width (for #width == 32 any value is accepted). + void WriteBits(ui32 value, int width); + + //! Pads the last partial byte with zero low bits and returns the + //! one-past-end pointer. + char* Finish(); + +private: + char* Ptr_; + ui64 Accumulator_ = 0; + int BitCount_ = 0; +}; + +//! MSB-first bit reader. +/*! + * Reads bits written by #TBitWriter. Assumes up to 8 bytes past the logical end + * of the stream are safe to read. + */ +class TBitReader +{ +public: + explicit TBitReader(const char* ptr); + + //! Reads and returns #width bits (0 <= #width <= 32). + ui32 ReadBits(int width); + + //! Returns the one-past-end pointer; the (< 8) buffered sub-byte bits, which + //! are the writer's zero padding, are dropped. + const char* Finish(); + +private: + const ui8* Ptr_; + ui64 Accumulator_ = 0; + int BitCount_ = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define BIT_IO_INL_H_ +#include "bit_io-inl.h" +#undef BIT_IO_INL_H_ diff --git a/library/cpp/yt/coding/interpolative-inl.h b/library/cpp/yt/coding/interpolative-inl.h new file mode 100644 index 00000000000..5eadfdd483e --- /dev/null +++ b/library/cpp/yt/coding/interpolative-inl.h @@ -0,0 +1,182 @@ +#ifndef INTERPOLATIVE_INL_H_ +#error "Direct inclusion of this file is not allowed, include interpolative.h" +// For the sake of sane code completion. +#include "interpolative.h" +#endif + +#include "bit_io.h" + +#include <library/cpp/yt/assert/assert.h> + +#include <library/cpp/yt/memory/range.h> + +#include <util/system/compiler.h> + +#include <array> +#include <bit> +#include <concepts> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NInterpolativeCodingDetail { + +// Truncated-binary (minimal) code for a value in [0, rangeSize): the first +// Cutoff values take LowWidth bits, the rest one more. +struct TTruncatedBinaryParams +{ + int LowWidth; // floor(log2(rangeSize)) + ui32 Cutoff; // 2^(LowWidth + 1) - rangeSize +}; + +Y_FORCE_INLINE TTruncatedBinaryParams GetTruncatedBinaryParams(ui32 rangeSize) +{ + int lowWidth = std::bit_width(rangeSize) - 1; + ui32 cutoff = (2u << lowWidth) - rangeSize; + return {lowWidth, cutoff}; +} + +// Writes #value in [0, #rangeSize) with the entropy-optimal integer code for a +// uniform value. rangeSize == 1 yields lowWidth 0 / cutoff 1 and emits a +// zero-width code (a no-op), so the singleton case needs no branch. +Y_FORCE_INLINE void WriteTruncatedBinary(TBitWriter* writer, ui32 value, ui32 rangeSize) +{ + auto [lowWidth, cutoff] = GetTruncatedBinaryParams(rangeSize); + // Branchless: the (value >= cutoff) predicate would mispredict on nearly + // every element, so fold it into the emitted codeword and width instead. + ui32 isLong = value >= cutoff ? 1 : 0; + writer->WriteBits(value + (cutoff & (0u - isLong)), lowWidth + static_cast<int>(isLong)); +} + +Y_FORCE_INLINE ui32 ReadTruncatedBinary(TBitReader* reader, ui32 rangeSize) +{ + auto [lowWidth, cutoff] = GetTruncatedBinaryParams(rangeSize); + ui32 high = reader->ReadBits(lowWidth); + if (high < cutoff) { + return high; + } + ui32 low = reader->ReadBits(1); + return ((high << 1) | low) - cutoff; +} + +// A subrange [BeginIndex, EndIndex) of the value array together with the +// inclusive value bounds [Lo, Hi] that its elements must fall in. The median +// values[m] is the only array element touched per node, which matters because +// the traversal walks a large sequence in a cache-unfriendly tree order. +struct TInterpolativeFrame +{ + int BeginIndex; + int EndIndex; + ui32 Lo; + ui32 Hi; +}; + +} // namespace NInterpolativeCodingDetail + +//////////////////////////////////////////////////////////////////////////////// + +inline size_t GetInterpolativeMaxByteSize(int count, ui32 lo, ui32 hi) +{ + // Each element is coded in at most ceil(log2(hi - lo + 1)) bits; the trailing + // word covers the writer's 4-byte flush store. + int maxBitWidth = std::bit_width(hi - lo); + return (static_cast<size_t>(count) * maxBitWidth + 7) / 8 + sizeof(ui32); +} + +//////////////////////////////////////////////////////////////////////////////// + +// Both traversals descend the left child in place and stack only the (non-empty) +// right child, so the stack sees ~count/2 pushes instead of ~count. Left-first +// order matches between encoder and decoder. +template <std::unsigned_integral T> +void InterpolativeEncode(TBitWriter* writer, TRange<T> values, ui32 lo, ui32 hi) +{ + using namespace NInterpolativeCodingDetail; + + int count = std::ssize(values); + if (count == 0) { + return; + } + + // Right children stack up along the leftmost path => depth <= ceil(log2(count)). + std::array<TInterpolativeFrame, 48> stack; + int top = 0; + int beginIndex = 0; + int endIndex = count; + ui32 l = lo; + ui32 h = hi; + for (;;) { + // beginIndex is invariant while descending left; only endIndex/l/h change. + while (beginIndex < endIndex) { + int half = (endIndex - beginIndex) / 2; + int m = beginIndex + half; + // The next descent step reads values[beginIndex + half/2]; prefetch it + // to hide the cache-scattered tree walk on large sequences. + Y_PREFETCH_READ(values.data() + beginIndex + (half >> 1), 0); + ui32 value = static_cast<ui32>(values[m]); + // rangeSize = (upperBound - lowerBound + 1) simplifies to this; + // lowerBound = l + half. + ui32 rangeSize = (h - l) - static_cast<ui32>(endIndex - beginIndex) + 2; + WriteTruncatedBinary(writer, value - l - static_cast<ui32>(half), rangeSize); + if (m + 1 < endIndex) { + YT_ASSERT(top < std::ssize(stack)); + stack[top++] = {m + 1, endIndex, value + 1, h}; + } + endIndex = m; + h = value - 1; + } + if (top == 0) { + break; + } + auto f = stack[--top]; + beginIndex = f.BeginIndex; + endIndex = f.EndIndex; + l = f.Lo; + h = f.Hi; + } +} + +template <std::unsigned_integral T> +void InterpolativeDecode(TBitReader* reader, TMutableRange<T> values, ui32 lo, ui32 hi) +{ + using namespace NInterpolativeCodingDetail; + + int count = std::ssize(values); + if (count == 0) { + return; + } + + std::array<TInterpolativeFrame, 48> stack; + int top = 0; + int beginIndex = 0; + int endIndex = count; + ui32 l = lo; + ui32 h = hi; + for (;;) { + while (beginIndex < endIndex) { + int half = (endIndex - beginIndex) / 2; + int m = beginIndex + half; + ui32 rangeSize = (h - l) - static_cast<ui32>(endIndex - beginIndex) + 2; + ui32 value = l + static_cast<ui32>(half) + ReadTruncatedBinary(reader, rangeSize); + values[m] = static_cast<T>(value); + if (m + 1 < endIndex) { + stack[top++] = {m + 1, endIndex, value + 1, h}; + } + endIndex = m; + h = value - 1; + } + if (top == 0) { + break; + } + auto f = stack[--top]; + beginIndex = f.BeginIndex; + endIndex = f.EndIndex; + l = f.Lo; + h = f.Hi; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/coding/interpolative.h b/library/cpp/yt/coding/interpolative.h new file mode 100644 index 00000000000..88e415e754c --- /dev/null +++ b/library/cpp/yt/coding/interpolative.h @@ -0,0 +1,52 @@ +#pragma once + +#include "bit_io.h" + +#include <library/cpp/yt/memory/range.h> + +#include <util/system/types.h> + +#include <concepts> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// +// +// Binary interpolative coding for a sorted, strictly increasing sequence of +// integers drawn from a known range [lo, hi]. The value domain is 32-bit: lo, hi +// and hence every value fit in ui32, independent of the element type T (which is +// merely the container's width). +// +// It recursively encodes the median element within the range implied by its +// position and its already-coded neighbors, so clustered sequences compress far +// below a flat log2 per element and no per-element headers are needed. Each +// element is stored with a truncated-binary (minimal) code, which spends the +// fractional part of log2(range) instead of rounding every element up to a whole +// bit. +// +// The bit stream is MSB-first (see bit_io.h). +// +//////////////////////////////////////////////////////////////////////////////// + +//! Encodes #values, which must be strictly increasing and all within [#lo, #hi], +//! with binary interpolative coding. An empty range emits nothing; the length is +//! not stored and must be conveyed out of band (e.g. as a varint prefix). +template <std::unsigned_integral T> +void InterpolativeEncode(TBitWriter* writer, TRange<T> values, ui32 lo, ui32 hi); + +//! Decodes a sequence written by #InterpolativeEncode into #values, whose size +//! must equal the encoded element count. #lo and #hi must match the encoder. +template <std::unsigned_integral T> +void InterpolativeDecode(TBitReader* reader, TMutableRange<T> values, ui32 lo, ui32 hi); + +//! An upper bound on the buffer size #InterpolativeEncode needs to encode #count +//! values over [#lo, #hi], including the slack the writer requires. +size_t GetInterpolativeMaxByteSize(int count, ui32 lo, ui32 hi); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define INTERPOLATIVE_INL_H_ +#include "interpolative-inl.h" +#undef INTERPOLATIVE_INL_H_ diff --git a/library/cpp/yt/coding/unittests/bit_io_ut.cpp b/library/cpp/yt/coding/unittests/bit_io_ut.cpp new file mode 100644 index 00000000000..5d61b166f0a --- /dev/null +++ b/library/cpp/yt/coding/unittests/bit_io_ut.cpp @@ -0,0 +1,69 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/coding/bit_io.h> + +#include <utility> +#include <vector> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TBitIOTest, RoundTrip) +{ + std::vector<std::pair<ui32, int>> items = { + {0, 0}, {1, 1}, {0, 1}, {5, 3}, {0, 8}, {255, 8}, {12345, 14}, + {0, 32}, {0xffffffffu, 32}, {0x12345, 17}, {7, 3}, {1u << 30, 31}, + }; + std::vector<char> buffer(256, 0); + + TBitWriter writer(buffer.data()); + for (auto [value, width] : items) { + writer.WriteBits(value, width); + } + writer.Finish(); + + TBitReader reader(buffer.data()); + for (auto [value, width] : items) { + EXPECT_EQ(reader.ReadBits(width), value) << "width=" << width; + } +} + +TEST(TBitIOTest, ZeroWidthIsNoop) +{ + std::vector<char> buffer(16, 0); + TBitWriter writer(buffer.data()); + writer.WriteBits(0, 0); + writer.WriteBits(1, 1); + writer.WriteBits(0, 0); + char* end = writer.Finish(); + EXPECT_EQ(end - buffer.data(), 1); // a single bit occupies one byte + + TBitReader reader(buffer.data()); + EXPECT_EQ(reader.ReadBits(0), 0u); + EXPECT_EQ(reader.ReadBits(1), 1u); + EXPECT_EQ(reader.ReadBits(0), 0u); +} + +TEST(TBitIOTest, FlushBoundary) +{ + // Many 17-bit writes repeatedly cross the internal 32-bit flush boundary. + constexpr int Count = 1000; + std::vector<char> buffer(Count * 3 + 16, 0); + TBitWriter writer(buffer.data()); + for (int i = 0; i < Count; ++i) { + writer.WriteBits(static_cast<ui32>(i) & 0x1ffff, 17); + } + writer.Finish(); + + TBitReader reader(buffer.data()); + for (int i = 0; i < Count; ++i) { + EXPECT_EQ(reader.ReadBits(17), static_cast<ui32>(i) & 0x1ffff) << "i=" << i; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/coding/unittests/interpolative_ut.cpp b/library/cpp/yt/coding/unittests/interpolative_ut.cpp new file mode 100644 index 00000000000..a63b12f6eac --- /dev/null +++ b/library/cpp/yt/coding/unittests/interpolative_ut.cpp @@ -0,0 +1,221 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/coding/interpolative.h> +#include <library/cpp/yt/coding/varint.h> + +#include <algorithm> +#include <bit> +#include <numeric> +#include <random> +#include <set> +#include <vector> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// +// Truncated binary + +TEST(TTruncatedBinaryTest, Exhaustive) +{ + for (ui32 rangeSize = 1; rangeSize <= 2050; ++rangeSize) { + for (ui32 value = 0; value < rangeSize; ++value) { + std::vector<char> buffer(16, 0); + TBitWriter writer(buffer.data()); + NInterpolativeCodingDetail::WriteTruncatedBinary(&writer, value, rangeSize); + writer.Finish(); + + TBitReader reader(buffer.data()); + EXPECT_EQ(NInterpolativeCodingDetail::ReadTruncatedBinary(&reader, rangeSize), value) + << "rangeSize=" << rangeSize << " value=" << value; + } + } +} + +TEST(TTruncatedBinaryTest, MinimalLength) +{ + // Every codeword is floor(log2(rangeSize)) or ceil(log2(rangeSize)) bits. + for (ui32 rangeSize = 1; rangeSize <= 1000; ++rangeSize) { + int lowWidth = std::bit_width(rangeSize) - 1; + for (ui32 value = 0; value < rangeSize; ++value) { + std::vector<char> buffer(16, 0); + TBitWriter writer(buffer.data()); + NInterpolativeCodingDetail::WriteTruncatedBinary(&writer, value, rangeSize); + char* end = writer.Finish(); + + TBitReader reader(buffer.data()); + NInterpolativeCodingDetail::ReadTruncatedBinary(&reader, rangeSize); + const char* readEnd = reader.Finish(); + i64 bytes = end - buffer.data(); + // Read must consume no more bytes than were written. + EXPECT_LE(readEnd - buffer.data(), bytes); + EXPECT_LE(bytes, (lowWidth + 1 + 7) / 8 + 1); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Interpolative coding + +template <class T> +std::vector<char> Encode(const std::vector<T>& values, ui32 lo, ui32 hi) +{ + std::vector<char> buffer(values.size() * sizeof(ui32) + 16, 0); + TBitWriter writer(buffer.data()); + InterpolativeEncode(&writer, TRange(values), lo, hi); + char* end = writer.Finish(); + buffer.resize(end - buffer.data()); + return buffer; +} + +template <class T> +std::vector<T> Decode(std::vector<char> buffer, int count, ui32 lo, ui32 hi) +{ + buffer.resize(buffer.size() + 8, 0); // reader may over-read up to 8 bytes + std::vector<T> values(count); + TBitReader reader(buffer.data()); + InterpolativeDecode(&reader, TMutableRange(values), lo, hi); + return values; +} + +template <class T> +void ExpectRoundTrip(const std::vector<T>& values, ui32 lo, ui32 hi) +{ + auto decoded = Decode<T>(Encode(values, lo, hi), std::ssize(values), lo, hi); + EXPECT_EQ(decoded, values); +} + +TEST(TInterpolativeCodingTest, Empty) +{ + ExpectRoundTrip<ui32>({}, 0, 100); +} + +TEST(TInterpolativeCodingTest, Single) +{ + ExpectRoundTrip<ui32>({0}, 0, 0); + ExpectRoundTrip<ui32>({42}, 0, 100); + ExpectRoundTrip<ui32>({100}, 0, 100); +} + +TEST(TInterpolativeCodingTest, SmallCases) +{ + ExpectRoundTrip<ui32>({3, 7}, 0, 10); + ExpectRoundTrip<ui32>({0, 1, 2}, 0, 2); // full, zero bits + ExpectRoundTrip<ui32>({0, 5, 10}, 0, 10); // boundaries present + ExpectRoundTrip<ui32>({1, 2, 3, 4, 5}, 0, 6); +} + +TEST(TInterpolativeCodingTest, FullRange) +{ + // Every value present => every range collapses to a singleton (zero bits). + std::vector<ui32> values(500); + std::iota(values.begin(), values.end(), 7); + auto encoded = Encode(values, 7, 506); + EXPECT_TRUE(encoded.empty()); + EXPECT_EQ(Decode<ui32>(encoded, 500, 7, 506), values); +} + +TEST(TInterpolativeCodingTest, RandomRoundTrip) +{ + std::mt19937 rng(12345); + for (int iteration = 0; iteration < 500; ++iteration) { + ui32 universe = 1 + rng() % 200'000; + int count = std::min<ui32>(universe, 1 + rng() % 2000); + std::set<ui32> unique; + std::uniform_int_distribution<ui32> dist(0, universe - 1); + while (std::ssize(unique) < count) { + unique.insert(dist(rng)); + } + std::vector<ui32> values(unique.begin(), unique.end()); + ExpectRoundTrip<ui32>(values, 0, universe - 1); + } +} + +TEST(TInterpolativeCodingTest, NonZeroLowerBound) +{ + std::mt19937 rng(999); + for (int iteration = 0; iteration < 200; ++iteration) { + ui32 lo = rng() % 100'000; + ui32 span = 1 + rng() % 100'000; + ui32 hi = lo + span; + int count = std::min<ui32>(span + 1, 1 + rng() % 500); + std::set<ui32> unique; + std::uniform_int_distribution<ui32> dist(lo, hi); + while (std::ssize(unique) < count) { + unique.insert(dist(rng)); + } + std::vector<ui32> values(unique.begin(), unique.end()); + ExpectRoundTrip<ui32>(values, lo, hi); + } +} + +TEST(TInterpolativeCodingTest, Ui64Values) +{ + std::vector<ui64> values = {0, 1, 100, 1000, 50'000, 200'000}; + ExpectRoundTrip<ui64>(values, 0, 200'000); +} + +TEST(TInterpolativeCodingTest, MaxByteSize) +{ + std::mt19937 rng(555); + for (int iteration = 0; iteration < 300; ++iteration) { + ui32 universe = 1 + rng() % 200'000; + int count = std::min<ui32>(universe, 1 + rng() % 1000); + std::set<ui32> unique; + std::uniform_int_distribution<ui32> dist(0, universe - 1); + while (std::ssize(unique) < count) { + unique.insert(dist(rng)); + } + std::vector<ui32> values(unique.begin(), unique.end()); + + size_t bound = GetInterpolativeMaxByteSize(count, 0, universe - 1); + std::vector<char> buffer(bound, 0); + TBitWriter writer(buffer.data()); + InterpolativeEncode(&writer, TRange(values), 0, universe - 1); + EXPECT_LE(static_cast<size_t>(writer.Finish() - buffer.data()), bound); + } +} + +TEST(TInterpolativeCodingTest, MultipleListsInOneBuffer) +{ + // Mirrors real usage: each list is prefixed with a varint length and is + // byte-aligned, so lists can be concatenated and read back in sequence. + std::mt19937 rng(7); + ui32 hi = 199999; + std::vector<std::vector<ui32>> lists; + for (int listIndex = 0; listIndex < 50; ++listIndex) { + int count = 1 + rng() % 300; + std::set<ui32> unique; + std::uniform_int_distribution<ui32> dist(0, hi); + while (std::ssize(unique) < count) { + unique.insert(dist(rng)); + } + lists.emplace_back(unique.begin(), unique.end()); + } + + std::vector<char> buffer(200'000, 0); + char* ptr = buffer.data(); + for (const auto& list : lists) { + ptr += WriteVarUint32(ptr, static_cast<ui32>(list.size())); + TBitWriter writer(ptr); + InterpolativeEncode(&writer, TRange(list), 0, hi); + ptr = writer.Finish(); + } + + const char* readPtr = buffer.data(); + for (const auto& list : lists) { + ui32 count; + readPtr += ReadVarUint32(readPtr, &count); + ASSERT_EQ(count, list.size()); + std::vector<ui32> decoded(count); + TBitReader reader(readPtr); + InterpolativeDecode(&reader, TMutableRange(decoded), 0, hi); + readPtr = reader.Finish(); + EXPECT_EQ(decoded, list); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/coding/unittests/ya.make b/library/cpp/yt/coding/unittests/ya.make index ab94ee8796a..945e63ab506 100644 --- a/library/cpp/yt/coding/unittests/ya.make +++ b/library/cpp/yt/coding/unittests/ya.make @@ -3,6 +3,8 @@ GTEST() INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) SRCS( + bit_io_ut.cpp + interpolative_ut.cpp zig_zag_ut.cpp varint_ut.cpp ) diff --git a/library/cpp/yt/coding/ya.make b/library/cpp/yt/coding/ya.make index 639d94e755c..e5ea301cf90 100644 --- a/library/cpp/yt/coding/ya.make +++ b/library/cpp/yt/coding/ya.make @@ -6,7 +6,9 @@ SRCS( ) PEERDIR( + library/cpp/yt/assert library/cpp/yt/exception + library/cpp/yt/memory ) END() diff --git a/library/cpp/yt/coding/zig_zag-inl.h b/library/cpp/yt/coding/zig_zag-inl.h index c611f7e1d46..9df9a2cc5f4 100644 --- a/library/cpp/yt/coding/zig_zag-inl.h +++ b/library/cpp/yt/coding/zig_zag-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "zig_zag.h" #endif -#undef ZIG_ZAG_INL_H_ namespace NYT { diff --git a/library/cpp/yt/compact_containers/compact_vector-inl.h b/library/cpp/yt/compact_containers/compact_vector-inl.h index cec19960e19..1070e6b1c91 100644 --- a/library/cpp/yt/compact_containers/compact_vector-inl.h +++ b/library/cpp/yt/compact_containers/compact_vector-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "compact_vector.h" #endif -#undef COMPACT_VECTOR_INL_H_ #include <library/cpp/yt/assert/assert.h> diff --git a/library/cpp/yt/error/error.cpp b/library/cpp/yt/error/error.cpp index d0a5489ab22..dc5aca7ccd2 100644 --- a/library/cpp/yt/error/error.cpp +++ b/library/cpp/yt/error/error.cpp @@ -5,6 +5,8 @@ #include <library/cpp/yt/error/error_attributes.h> #include <library/cpp/yt/error/origin_attributes.h> +#include <library/cpp/yt/memory/leaky_singleton.h> + #include <library/cpp/yt/string/string.h> #include <library/cpp/yt/system/proc.h> @@ -29,8 +31,20 @@ void FormatValue(TStringBuilderBase* builder, TErrorCode code, TStringBuf spec) constexpr TStringBuf ErrorMessageTruncatedSuffix = "...<message truncated>"; -TError::TEnricher TError::Enricher_; -TError::TFromExceptionEnricher TError::FromExceptionEnricher_; +namespace { + +struct TEnricherStorage +{ + static TEnricherStorage* Get() + { + return LeakySingleton<TEnricherStorage>(); + } + + TError::TEnricher Enricher; + TError::TFromExceptionEnricher FromExceptionEnricher; +}; + +} // namespace //////////////////////////////////////////////////////////////////////////////// @@ -636,11 +650,12 @@ void TError::RegisterEnricher(TEnricher enricher) { // NB: This daisy-chaining strategy is optimal when there's O(1) callbacks. Convert to a vector // if the number grows. - if (!Enricher_) { - Enricher_ = std::move(enricher); + auto* storage = TEnricherStorage::Get(); + if (!storage->Enricher) { + storage->Enricher = std::move(enricher); return; } - Enricher_ = [first = std::move(Enricher_), second = std::move(enricher)] (TError* error) { + storage->Enricher = [first = std::move(storage->Enricher), second = std::move(enricher)] (TError* error) { first(error); second(error); }; @@ -650,12 +665,13 @@ void TError::RegisterFromExceptionEnricher(TFromExceptionEnricher enricher) { // NB: This daisy-chaining strategy is optimal when there's O(1) callbacks. Convert to a vector // if the number grows. - if (!FromExceptionEnricher_) { - FromExceptionEnricher_ = std::move(enricher); + auto* storage = TEnricherStorage::Get(); + if (!storage->FromExceptionEnricher) { + storage->FromExceptionEnricher = std::move(enricher); return; } - FromExceptionEnricher_ = [ - first = std::move(FromExceptionEnricher_), + storage->FromExceptionEnricher = [ + first = std::move(storage->FromExceptionEnricher), second = std::move(enricher) ] (TError* error, const std::exception& exception) { first(error, exception); @@ -676,15 +692,15 @@ void TError::MakeMutable() void TError::Enrich() { - if (Enricher_) { - Enricher_(this); + if (const auto& enricher = TEnricherStorage::Get()->Enricher) { + enricher(this); } } void TError::EnrichFromException(const std::exception& exception) { - if (FromExceptionEnricher_) { - FromExceptionEnricher_(this, exception); + if (const auto& enricher = TEnricherStorage::Get()->FromExceptionEnricher) { + enricher(this, exception); } } diff --git a/library/cpp/yt/error/error.h b/library/cpp/yt/error/error.h index b2d0abacfe8..13223af8d6c 100644 --- a/library/cpp/yt/error/error.h +++ b/library/cpp/yt/error/error.h @@ -253,9 +253,6 @@ private: void EnrichFromException(const std::exception& exception); friend class TErrorAttributes; - - static TEnricher Enricher_; - static TFromExceptionEnricher FromExceptionEnricher_; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h index 87672ec6f6a..fa20fb06489 100644 --- a/library/cpp/yt/logging/logger-inl.h +++ b/library/cpp/yt/logging/logger-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "logger.h" #endif -#undef LOGGER_INL_H_ #include <library/cpp/yt/yson_string/convert.h> #include <library/cpp/yt/yson_string/string.h> diff --git a/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h b/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h index d91d20f2fd6..4211806880f 100644 --- a/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h +++ b/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "atomic_intrusive_ptr.h" #endif -#undef ATOMIC_INTRUSIVE_PTR_INL_H_ #include <util/system/spinlock.h> diff --git a/library/cpp/yt/system/cpu_id-inl.h b/library/cpp/yt/system/cpu_id-inl.h index e336ba5a2ce..804995f5973 100644 --- a/library/cpp/yt/system/cpu_id-inl.h +++ b/library/cpp/yt/system/cpu_id-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "cpu_id.h" #endif -#undef CPU_ID_INL_H_ #ifdef __linux__ #include <library/cpp/yt/rseq/rseq.h> diff --git a/library/cpp/yt/threading/event_count-inl.h b/library/cpp/yt/threading/event_count-inl.h index 18f7bfec52f..b366f93bb0f 100644 --- a/library/cpp/yt/threading/event_count-inl.h +++ b/library/cpp/yt/threading/event_count-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "event_count.h" #endif -#undef EVENT_COUNT_INL_H_ #include <library/cpp/yt/assert/assert.h> diff --git a/library/cpp/yt/threading/recursive_spin_lock-inl.h b/library/cpp/yt/threading/recursive_spin_lock-inl.h index f692999f0b1..28b233efa94 100644 --- a/library/cpp/yt/threading/recursive_spin_lock-inl.h +++ b/library/cpp/yt/threading/recursive_spin_lock-inl.h @@ -3,7 +3,6 @@ // For the sake of sane code completion. #include "recursive_spin_lock.h" #endif -#undef RECURSIVE_SPIN_LOCK_INL_H_ #include "spin_wait.h" diff --git a/library/cpp/yt/threading/rw_spin_lock-inl.h b/library/cpp/yt/threading/rw_spin_lock-inl.h index c6043e8f23a..167c95edfc5 100644 --- a/library/cpp/yt/threading/rw_spin_lock-inl.h +++ b/library/cpp/yt/threading/rw_spin_lock-inl.h @@ -4,7 +4,6 @@ // For the sake of sane code completion. #include "rw_spin_lock.h" #endif -#undef RW_SPIN_LOCK_INL_H_ #include "spin_wait.h" diff --git a/library/cpp/yt/threading/spin_lock-inl.h b/library/cpp/yt/threading/spin_lock-inl.h index eedb1e52856..7e629ae3381 100644 --- a/library/cpp/yt/threading/spin_lock-inl.h +++ b/library/cpp/yt/threading/spin_lock-inl.h @@ -4,7 +4,6 @@ // For the sake of sane code completion. #include "spin_lock.h" #endif -#undef SPIN_LOCK_INL_H_ #include "spin_wait.h" diff --git a/library/cpp/yt/threading/spin_lock_base-inl.h b/library/cpp/yt/threading/spin_lock_base-inl.h index 7135d081112..2e16a4e01f9 100644 --- a/library/cpp/yt/threading/spin_lock_base-inl.h +++ b/library/cpp/yt/threading/spin_lock_base-inl.h @@ -4,7 +4,6 @@ // For the sake of sane code completion. #include "spin_lock_base.h" #endif -#undef SPIN_LOCK_BASE_INL_H_ #include <library/cpp/yt/misc/source_location.h> diff --git a/library/cpp/yt/threading/spin_lock_count-inl.h b/library/cpp/yt/threading/spin_lock_count-inl.h index 6ba1782052f..8100f2f3e73 100644 --- a/library/cpp/yt/threading/spin_lock_count-inl.h +++ b/library/cpp/yt/threading/spin_lock_count-inl.h @@ -2,7 +2,6 @@ #ifndef SPIN_LOCK_COUNT_INL_H_ #error "Direct inclusion of this file is not allowed, include spin_lock_count.h" #endif -#undef SPIN_LOCK_COUNT_INL_H_ #include <util/system/compiler.h> diff --git a/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h b/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h index b880135f244..590e3238a9d 100644 --- a/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h +++ b/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h @@ -4,7 +4,6 @@ // For the sake of sane code completion. #include "writer_starving_rw_spin_lock.h" #endif -#undef WRITER_STARVING_RW_SPIN_LOCK_INL_H_ #include "spin_wait.h" diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp index 696da0ebaec..f06b35f9bdb 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp @@ -138,7 +138,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no }; // adds _yql_sys_tablekeyswitch column which is required for outer lambda - // _yql_sys_tableswitch equals "true" when reduce key is changed + // _yql_sys_tablekeyswitch equals "true" when reduce key is changed TExprNode::TPtr keySwitchLambda = ctx.Builder(node.Pos()) .Lambda() .Param("stream") @@ -564,8 +564,6 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduceWithTrivialMa .Reducer(newReduceLambda) .Settings(newSettings) .Done(); - - return node; } TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseInnerMap(TExprBase node, TExprContext& ctx, const TGetParents& getParents) const { diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp index f81ee580f99..6c241b5c8dc 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp @@ -61,7 +61,7 @@ TYtSectionList ConvertInputTable(TExprBase input, TExprContext& ctx, const TConv mergedSettings = NYql::RemoveSetting(*mergedSettings, EYtSettingType::Unordered, ctx); makeUnordered = false; } - if (!opts.KeepDirecRead_) { + if (!opts.KeepDirectRead_) { mergedSettings = NYql::RemoveSetting(*mergedSettings, EYtSettingType::DirectRead, ctx); } if (opts.Settings_) { @@ -775,7 +775,7 @@ TExprBase GetWorld(TExprBase input, TMaybeNode<TExprBase> main, TExprContext& ct } TConvertInputOpts::TConvertInputOpts() - : KeepDirecRead_(false) + : KeepDirectRead_(false) , MakeUnordered_(false) , ClearUnordered_(false) { @@ -817,8 +817,8 @@ TConvertInputOpts::ExplicitFields(const TStructExprType& type, TPositionHandle p } TConvertInputOpts& -TConvertInputOpts::KeepDirecRead(bool keepDirecRead) { - KeepDirecRead_ = keepDirecRead; +TConvertInputOpts::KeepDirectRead(bool keepDirectRead) { + KeepDirectRead_ = keepDirectRead; return *this; } diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h index 04430f7ac23..291638037a1 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h @@ -57,7 +57,7 @@ NNodes::TExprBase GetWorld(NNodes::TExprBase input, NNodes::TMaybeNode<NNodes::T struct TConvertInputOpts { NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> Settings_; NNodes::TMaybeNode<NNodes::TCoAtomList> CustomFields_; - bool KeepDirecRead_; + bool KeepDirectRead_; bool MakeUnordered_; bool ClearUnordered_; @@ -71,7 +71,7 @@ struct TConvertInputOpts { TConvertInputOpts& ExplicitFields(const TStructExprType& type, TPositionHandle pos, TExprContext& ctx); - TConvertInputOpts& KeepDirecRead(bool keepDirecRead = true); + TConvertInputOpts& KeepDirectRead(bool keepDirectRead = true); TConvertInputOpts& MakeUnordered(bool makeUnordered = true); diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp index 8bf71c90547..02d5b8daab0 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp @@ -388,7 +388,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::TakeOrSkip(TExprBase no .Input<TYtReadTable>() .World(ApplySyncListToWorld(GetWorld(input, {}, ctx).Ptr(), syncList, ctx)) .DataSource(GetDataSource(input, ctx)) - .Input(ConvertInputTable(input, ctx, TConvertInputOpts().KeepDirecRead(true).Settings(settings))) + .Input(ConvertInputTable(input, ctx, TConvertInputOpts().KeepDirectRead(true).Settings(settings))) .Build() .Done(); return KeepColumnOrder(res.Ptr(), node.Ref(), ctx, *State_->Types); @@ -1003,7 +1003,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::PushPruneKeysIntoYtOper State_->Configuration->RuntimeClusterSelection.Get().GetOrElse(DEFAULT_RUNTIME_CLUSTER_SELECTION); auto cluster = DeriveClusterFromInput(op.Input(), selectionMode); if (!cluster || !IsYtCompleteIsolatedLambda(extractorLambda.Ref(), syncList, *cluster, false, selectionMode)) { - return {}; + return node; } auto outItemType = SilentGetSequenceItemType(op.Input().Ref(), true); diff --git a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp index dda15abc3db..67288ff28ed 100644 --- a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp @@ -438,11 +438,11 @@ public: const auto enableDynamicStoreRead = ytState->Configuration->EnableDynamicStoreReadInDQ.Get().GetOrElse(false); ui64 chunksCount = 0ull; for (auto section: maybeRead.Cast().Input()) { - if (HasSettingsExcept(maybeRead.Cast().Input().Item(0).Settings().Ref(), DqReadSupportedSettings) || HasNonEmptyKeyFilter(maybeRead.Cast().Input().Item(0))) { + if (HasSettingsExcept(section.Settings().Ref(), DqReadSupportedSettings) || HasNonEmptyKeyFilter(section)) { TStringBuilder info; info << "unsupported path settings: "; - if (maybeRead.Cast().Input().Item(0).Settings().Size() > 0) { - for (auto& setting : maybeRead.Cast().Input().Item(0).Settings().Ref().Children()) { + if (section.Settings().Size() > 0) { + for (auto& setting : section.Settings().Ref().Children()) { if (setting->ChildrenSize() != 0) { info << setting->Child(0)->Content() << ","; } @@ -678,7 +678,7 @@ public: ++idx; continue; } - ui64 readSize = std::accumulate(res[idx].begin(), res[idx].end(), 0); + ui64 readSize = std::accumulate(res[idx].begin(), res[idx].end(), 0ull); ++idx; dataSizePerJob = Max(ui64(dataSizePerJob / *codecCpu), 10_KB); const ui64 parts = (readSize + dataSizePerJob - 1) / dataSizePerJob; diff --git a/yt/yql/providers/yt/provider/yql_yt_key.cpp b/yt/yql/providers/yt/provider/yql_yt_key.cpp index 54b5eaf2b16..f1aa1bf18d9 100644 --- a/yt/yql/providers/yt/provider/yql_yt_key.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_key.cpp @@ -187,6 +187,7 @@ bool TYtKey::Parse(const TExprNode& key, TExprContext& ctx, bool isOutput) { } else { ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), "Expected String or MrTableRange[Strict]")); + return false; } if (key.ChildrenSize() > 1) { @@ -225,16 +226,6 @@ bool TYtKey::Parse(const TExprNode& key, TExprContext& ctx, bool isOutput) { return false; } ExtraColumns = value; - } else if (tag.IsAtom("typeId")) { - if (Type != EType::View) { - ctx.AddError(TIssue(ctx.GetPosition(tag.Pos()), "Missed object id.")); - return false; - } - if (const auto viewNode = key.Child(i)->Child(1); - !(viewNode->IsCallable("String") && viewNode->ChildrenSize() == 1 && viewNode->Head().IsAtom("VIEW"))) { - ctx.AddError(TIssue(ctx.GetPosition(viewNode->Pos()), "Expected String 'VIEW'.")); - return false; - } } else { ctx.AddError(TIssue(ctx.GetPosition(tag.Pos()), TStringBuilder() << "Unexpected tag: " << tag.Content())); return false; diff --git a/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp b/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp index a2afabd6b84..e701cd14979 100644 --- a/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp @@ -11,7 +11,12 @@ public: TVector<std::pair<TKey, const TLayerInfo*>> infos; infos.reserve(order.size()); for (auto &e: order) { - infos.emplace_back(e, Data_.FindPtr(e)); + auto info = Data_.FindPtr(e); + if (!info) { + ReportUnknownLayer(e, ctx); + return {}; + } + infos.emplace_back(e, info); if (infos.back().second->Locations.empty()) { ctx.AddError(NYql::TIssue( TStringBuilder() << "No locations for layer " << e.ToString() << ", consider adding it using yt.LayerCaches" @@ -41,10 +46,7 @@ public: bool UpdateLayerLocations(const TKey& key, TVector<TLocation>&& locs, NYql::TExprContext& ctx) override { auto ptr = Data_.FindPtr(key); if (!ptr) { - ctx.AddError(NYql::TIssue( - TStringBuilder() << "Layer with key=(Name=" << key.Name.GetOrElse("nullopt").Quote() - << ", Url=" << key.Url.GetOrElse("nullopt").Quote() << ") not found" - )); + ReportUnknownLayer(key, ctx); return false; } ptr->Locations = std::move(locs); @@ -56,6 +58,12 @@ public: } private: + static void ReportUnknownLayer(const TKey& key, NYql::TExprContext& ctx) { + ctx.AddError(NYql::TIssue( + TStringBuilder() << "Layer with key=(Name=" << key.Name.GetOrElse("nullopt").Quote() + << ", Url=" << key.Url.GetOrElse("nullopt").Quote() << ") not found" + )); + } THashMap<TKey, TLayerInfo> Data_; }; } diff --git a/yt/yt/core/yson/producer-inl.h b/yt/yt/core/yson/producer-inl.h index 7cfe5ff572c..034b972a234 100644 --- a/yt/yt/core/yson/producer-inl.h +++ b/yt/yt/core/yson/producer-inl.h @@ -11,8 +11,8 @@ namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// template <class... TAdditionalArgs> -TExtendedYsonProducer<TAdditionalArgs...>::TExtendedYsonProducer( - TExtendedYsonProducer::TUnderlyingCallback callback, +TParametricYsonProducer<TAdditionalArgs...>::TParametricYsonProducer( + TParametricYsonProducer::TUnderlyingCallback callback, EYsonType type) : Type_(type) , Callback_(std::move(callback)) @@ -21,11 +21,11 @@ TExtendedYsonProducer<TAdditionalArgs...>::TExtendedYsonProducer( } template <class... TAdditionalArgs> -void TExtendedYsonProducer<TAdditionalArgs...>::Run(IYsonConsumer* consumer, TAdditionalArgs... args) const +void TParametricYsonProducer<TAdditionalArgs...>::Run(IYsonConsumer* consumer, TAdditionalArgs... args) const { Callback_(consumer, std::forward<TAdditionalArgs>(args)...); } //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NYson
\ No newline at end of file +} // namespace NYT::NYson diff --git a/yt/yt/core/yson/producer.cpp b/yt/yt/core/yson/producer.cpp index aeff4b7266a..4951612d786 100644 --- a/yt/yt/core/yson/producer.cpp +++ b/yt/yt/core/yson/producer.cpp @@ -4,20 +4,6 @@ namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// -TYsonProducer::TYsonProducer(TYsonCallback callback, EYsonType type) - : Type_(type) - , Callback_(std::move(callback)) -{ - YT_ASSERT(Callback_); -} - -void TYsonProducer::Run(IYsonConsumer* consumer) const -{ - Callback_(consumer); -} - -//////////////////////////////////////////////////////////////////////////////// - void Serialize(const TYsonProducer& value, IYsonConsumer* consumer) { value.Run(consumer); diff --git a/yt/yt/core/yson/producer.h b/yt/yt/core/yson/producer.h index 7310d1540ba..5ddae14f540 100644 --- a/yt/yt/core/yson/producer.h +++ b/yt/yt/core/yson/producer.h @@ -12,45 +12,26 @@ namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// //! A callback capable of generating YSON by calling appropriate -//! methods for its IYsonConsumer argument. -using TYsonCallback = TCallback<void(IYsonConsumer*)>; - -//! A callback capable of generating YSON by calling appropriate //! methods for its IYsonConsumer and some additional arguments. template <class... TAdditionalArgs> -using TExtendedYsonCallback = TCallback<void(IYsonConsumer*, TAdditionalArgs...)>; - -//////////////////////////////////////////////////////////////////////////////// - -class TYsonProducer -{ -public: - DEFINE_BYVAL_RO_PROPERTY(NYson::EYsonType, Type); +using TParametricYsonCallback = TCallback<void(IYsonConsumer*, TAdditionalArgs...)>; -public: - TYsonProducer() = default; - TYsonProducer( - TYsonCallback callback, - EYsonType type = NYson::EYsonType::Node); - - void Run(IYsonConsumer* consumer) const; - -private: - TYsonCallback Callback_; -}; +//! A callback capable of generating YSON by calling appropriate +//! methods for its IYsonConsumer argument. +using TYsonCallback = TParametricYsonCallback<>; //////////////////////////////////////////////////////////////////////////////// template <class... TAdditionalArgs> -class TExtendedYsonProducer +class TParametricYsonProducer { - using TUnderlyingCallback = TExtendedYsonCallback<TAdditionalArgs...>; + using TUnderlyingCallback = TParametricYsonCallback<TAdditionalArgs...>; public: DEFINE_BYVAL_RO_PROPERTY(NYson::EYsonType, Type); public: - TExtendedYsonProducer() = default; - TExtendedYsonProducer( + TParametricYsonProducer() = default; + TParametricYsonProducer( TUnderlyingCallback callback, EYsonType type = NYson::EYsonType::Node); @@ -60,6 +41,9 @@ private: TUnderlyingCallback Callback_; }; +//! A producer generating a plain YSON with no additional arguments. +using TYsonProducer = TParametricYsonProducer<>; + //////////////////////////////////////////////////////////////////////////////// void Serialize(const TYsonProducer& value, NYson::IYsonConsumer* consumer); diff --git a/yt/yt/core/yson/public.h b/yt/yt/core/yson/public.h index b322e9007f1..70bd806d5a7 100644 --- a/yt/yt/core/yson/public.h +++ b/yt/yt/core/yson/public.h @@ -12,9 +12,9 @@ namespace NYT::NYson { enum class ETokenType; -class TYsonProducer; template <class... TAdditionalArgs> -class TExtendedYsonProducer; +class TParametricYsonProducer; +using TYsonProducer = TParametricYsonProducer<>; class TYsonInput; class TYsonOutput; diff --git a/yt/yt/core/ytree/ypath_service.cpp b/yt/yt/core/ytree/ypath_service.cpp index 3d3e4185617..16460f9b0cb 100644 --- a/yt/yt/core/ytree/ypath_service.cpp +++ b/yt/yt/core/ytree/ypath_service.cpp @@ -233,7 +233,7 @@ class TFromExtendedProducerYPathService : public TYPathServiceBase , public TSupportsGet { - using TUnderlyingProducer = TExtendedYsonProducer<const IAttributeDictionaryPtr&>; + using TUnderlyingProducer = TParametricYsonProducer<const IAttributeDictionaryPtr&>; public: explicit TFromExtendedProducerYPathService(TUnderlyingProducer producer) : Producer_(std::move(producer)) @@ -326,7 +326,7 @@ private: }; IYPathServicePtr IYPathService::FromProducer( - NYson::TExtendedYsonProducer<const IAttributeDictionaryPtr&> producer) + NYson::TParametricYsonProducer<const IAttributeDictionaryPtr&> producer) { return New<TFromExtendedProducerYPathService>(std::move(producer)); } diff --git a/yt/yt/core/ytree/ypath_service.h b/yt/yt/core/ytree/ypath_service.h index 6df44dba804..dbd655c2688 100644 --- a/yt/yt/core/ytree/ypath_service.h +++ b/yt/yt/core/ytree/ypath_service.h @@ -106,7 +106,7 @@ struct IYPathService * an ephemeral tree, and the request is forwarded to that tree. */ static IYPathServicePtr FromProducer( - NYson::TExtendedYsonProducer<const IAttributeDictionaryPtr&> producer); + NYson::TParametricYsonProducer<const IAttributeDictionaryPtr&> producer); //! Creates a producer from YPath service. /*! |
