summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYDBot <[email protected]>2026-07-13 03:53:22 +0000
committerYDBot <[email protected]>2026-07-13 03:53:22 +0000
commitc1fc37566f6a3639b67e7fc2e23eff1647893bd1 (patch)
tree189dcfc42f12b41b5119111d1452727a956a915e
parent778f44e34127879022a1e961ef7ca35d247981ab (diff)
parenta9010d99cb515f79a521fe7d26fb0c814936d972 (diff)
Merge pull request #46281 from ydb-platform/merge-rightlib-260713-0103
-rw-r--r--build/config/tests/flake8/flake8.conf11
-rw-r--r--build/external_resources/flake8_py3/ya.make10
-rw-r--r--build/mapping.conf.json10
-rw-r--r--contrib/python/s3transfer/py3/.dist-info/METADATA6
-rw-r--r--contrib/python/s3transfer/py3/s3transfer/__init__.py2
-rw-r--r--contrib/python/s3transfer/py3/s3transfer/crt.py10
-rw-r--r--contrib/python/s3transfer/py3/s3transfer/manager.py6
-rw-r--r--contrib/python/s3transfer/py3/tests/functional/test_crt.py12
-rw-r--r--contrib/python/s3transfer/py3/tests/functional/test_upload.py159
-rw-r--r--contrib/python/s3transfer/py3/ya.make2
-rw-r--r--contrib/python/scramp/.dist-info/METADATA12
-rw-r--r--contrib/python/scramp/README.md10
-rw-r--r--contrib/python/scramp/scramp/core.py25
-rw-r--r--contrib/python/scramp/scramp/utils.py2
-rw-r--r--contrib/python/scramp/ya.make2
-rw-r--r--library/cpp/yt/coding/bit_io-inl.h69
-rw-r--r--library/cpp/yt/coding/bit_io.h63
-rw-r--r--library/cpp/yt/coding/interpolative-inl.h182
-rw-r--r--library/cpp/yt/coding/interpolative.h52
-rw-r--r--library/cpp/yt/coding/unittests/bit_io_ut.cpp69
-rw-r--r--library/cpp/yt/coding/unittests/interpolative_ut.cpp221
-rw-r--r--library/cpp/yt/coding/unittests/ya.make2
-rw-r--r--library/cpp/yt/coding/ya.make2
-rw-r--r--library/cpp/yt/coding/zig_zag-inl.h1
-rw-r--r--library/cpp/yt/compact_containers/compact_vector-inl.h1
-rw-r--r--library/cpp/yt/error/error.cpp42
-rw-r--r--library/cpp/yt/error/error.h3
-rw-r--r--library/cpp/yt/logging/logger-inl.h1
-rw-r--r--library/cpp/yt/memory/atomic_intrusive_ptr-inl.h1
-rw-r--r--library/cpp/yt/system/cpu_id-inl.h1
-rw-r--r--library/cpp/yt/threading/event_count-inl.h1
-rw-r--r--library/cpp/yt/threading/recursive_spin_lock-inl.h1
-rw-r--r--library/cpp/yt/threading/rw_spin_lock-inl.h1
-rw-r--r--library/cpp/yt/threading/spin_lock-inl.h1
-rw-r--r--library/cpp/yt/threading/spin_lock_base-inl.h1
-rw-r--r--library/cpp/yt/threading/spin_lock_count-inl.h1
-rw-r--r--library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h1
-rw-r--r--yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp4
-rw-r--r--yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp8
-rw-r--r--yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h4
-rw-r--r--yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp4
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp8
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_key.cpp11
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp18
-rw-r--r--yt/yt/core/yson/producer-inl.h8
-rw-r--r--yt/yt/core/yson/producer.cpp14
-rw-r--r--yt/yt/core/yson/producer.h38
-rw-r--r--yt/yt/core/yson/public.h4
-rw-r--r--yt/yt/core/ytree/ypath_service.cpp4
-rw-r--r--yt/yt/core/ytree/ypath_service.h2
50 files changed, 956 insertions, 167 deletions
diff --git a/build/config/tests/flake8/flake8.conf b/build/config/tests/flake8/flake8.conf
index a4a60be7432..4977577208e 100644
--- a/build/config/tests/flake8/flake8.conf
+++ b/build/config/tests/flake8/flake8.conf
@@ -9,7 +9,7 @@ select =
#B, S, # bandit
#C, # commas
#D, # docstrings
- #P, # string-format
+ #FMT, # string-format
#Q, # quotes
ignore =
@@ -30,6 +30,15 @@ ignore =
# line break after binary operator
W504,
+ # format string contains unindexed parameters
+ FMT101,
+ # docstring contains unindexed parameters
+ FMT102,
+ # other string contains unindexed parameters
+ FMT103,
+ # format call uses implicit and explicit indexes together
+ FMT205,
+
# astroid-error
PLF002,
# function-redefined
diff --git a/build/external_resources/flake8_py3/ya.make b/build/external_resources/flake8_py3/ya.make
index 79961a5181a..2d50f6e09e9 100644
--- a/build/external_resources/flake8_py3/ya.make
+++ b/build/external_resources/flake8_py3/ya.make
@@ -12,12 +12,12 @@ ENDIF()
DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE(
FLAKE8_PY3
- sbr:10993120939 FOR DARWIN-ARM64
- sbr:10993122025 FOR DARWIN
+ sbr:12763243518 FOR DARWIN-ARM64
+ sbr:12763245515 FOR DARWIN
sbr:6726869647 FOR LINUX-PPC64LE
- sbr:10993123638 FOR LINUX
- sbr:10993119916 FOR LINUX-AARCH64
- sbr:10993122759 FOR WIN32
+ sbr:12763248856 FOR LINUX
+ sbr:12763242238 FOR LINUX-AARCH64
+ sbr:12763247016 FOR WIN32
)
END()
diff --git a/build/mapping.conf.json b/build/mapping.conf.json
index 3a396ebb3e8..4f6ac9c54fd 100644
--- a/build/mapping.conf.json
+++ b/build/mapping.conf.json
@@ -474,6 +474,7 @@
"6812259929": "{registry_endpoint}/6812259929",
"10531322967": "{registry_endpoint}/10531322967",
"10993122025": "{registry_endpoint}/10993122025",
+ "12763245515": "{registry_endpoint}/12763245515",
"6603784347": "{registry_endpoint}/6603784347",
"6652224583": "{registry_endpoint}/6652224583",
"6674734480": "{registry_endpoint}/6674734480",
@@ -485,6 +486,7 @@
"9245124509": "{registry_endpoint}/9245124509",
"10531314816": "{registry_endpoint}/10531314816",
"10993120939": "{registry_endpoint}/10993120939",
+ "12763243518": "{registry_endpoint}/12763243518",
"6603781997": "{registry_endpoint}/6603781997",
"6652223452": "{registry_endpoint}/6652223452",
"6674733966": "{registry_endpoint}/6674733966",
@@ -496,6 +498,7 @@
"9245123262": "{registry_endpoint}/9245123262",
"10531337045": "{registry_endpoint}/10531337045",
"10993123638": "{registry_endpoint}/10993123638",
+ "12763248856": "{registry_endpoint}/12763248856",
"6603787904": "{registry_endpoint}/6603787904",
"6652225714": "{registry_endpoint}/6652225714",
"6674734993": "{registry_endpoint}/6674734993",
@@ -507,6 +510,7 @@
"9245126434": "{registry_endpoint}/9245126434",
"10531306931": "{registry_endpoint}/10531306931",
"10993119916": "{registry_endpoint}/10993119916",
+ "12763242238": "{registry_endpoint}/12763242238",
"6603780916": "{registry_endpoint}/6603780916",
"6652222817": "{registry_endpoint}/6652222817",
"6674733653": "{registry_endpoint}/6674733653",
@@ -522,6 +526,7 @@
"6726869647": "{registry_endpoint}/6726869647",
"10531330161": "{registry_endpoint}/10531330161",
"10993122759": "{registry_endpoint}/10993122759",
+ "12763247016": "{registry_endpoint}/12763247016",
"6603785992": "{registry_endpoint}/6603785992",
"6652225128": "{registry_endpoint}/6652225128",
"6674734716": "{registry_endpoint}/6674734716",
@@ -2879,6 +2884,7 @@
"6812259929": "devtools/local_cache/toolscache/server/ya-tc for linux-aarch64",
"10531322967": "devtools/ya/test/programs/flake8/flake8 for darwin",
"10993122025": "devtools/ya/test/programs/flake8/flake8 for darwin",
+ "12763245515": "devtools/ya/test/programs/flake8/flake8 for darwin",
"6603784347": "devtools/ya/test/programs/flake8/flake8 for darwin",
"6652224583": "devtools/ya/test/programs/flake8/flake8 for darwin",
"6674734480": "devtools/ya/test/programs/flake8/flake8 for darwin",
@@ -2890,6 +2896,7 @@
"9245124509": "devtools/ya/test/programs/flake8/flake8 for darwin",
"10531314816": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"10993120939": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
+ "12763243518": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6603781997": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6652223452": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6674733966": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
@@ -2901,6 +2908,7 @@
"9245123262": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"10531337045": "devtools/ya/test/programs/flake8/flake8 for linux",
"10993123638": "devtools/ya/test/programs/flake8/flake8 for linux",
+ "12763248856": "devtools/ya/test/programs/flake8/flake8 for linux",
"6603787904": "devtools/ya/test/programs/flake8/flake8 for linux",
"6652225714": "devtools/ya/test/programs/flake8/flake8 for linux",
"6674734993": "devtools/ya/test/programs/flake8/flake8 for linux",
@@ -2912,6 +2920,7 @@
"9245126434": "devtools/ya/test/programs/flake8/flake8 for linux",
"10531306931": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"10993119916": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
+ "12763242238": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6603780916": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6652222817": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6674733653": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
@@ -2927,6 +2936,7 @@
"6726869647": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le",
"10531330161": "devtools/ya/test/programs/flake8/flake8 for win32",
"10993122759": "devtools/ya/test/programs/flake8/flake8 for win32",
+ "12763247016": "devtools/ya/test/programs/flake8/flake8 for win32",
"6603785992": "devtools/ya/test/programs/flake8/flake8 for win32",
"6652225128": "devtools/ya/test/programs/flake8/flake8 for win32",
"6674734716": "devtools/ya/test/programs/flake8/flake8 for win32",
diff --git a/contrib/python/s3transfer/py3/.dist-info/METADATA b/contrib/python/s3transfer/py3/.dist-info/METADATA
index 629c988c78a..cd598e2d609 100644
--- a/contrib/python/s3transfer/py3/.dist-info/METADATA
+++ b/contrib/python/s3transfer/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: s3transfer
-Version: 0.11.0
+Version: 0.11.3
Summary: An Amazon S3 Transfer Manager
Home-page: https://github.com/boto/s3transfer
Author: Amazon Web Services
@@ -22,9 +22,9 @@ Classifier: Programming Language :: Python :: 3.13
Requires-Python: >= 3.8
License-File: LICENSE.txt
License-File: NOTICE.txt
-Requires-Dist: botocore (<2.0a.0,>=1.33.2)
+Requires-Dist: botocore (<2.0a.0,>=1.36.0)
Provides-Extra: crt
-Requires-Dist: botocore[crt] (<2.0a.0,>=1.33.2) ; extra == 'crt'
+Requires-Dist: botocore[crt] (<2.0a.0,>=1.36.0) ; extra == 'crt'
=====================================================
s3transfer - An Amazon S3 Transfer Manager for Python
diff --git a/contrib/python/s3transfer/py3/s3transfer/__init__.py b/contrib/python/s3transfer/py3/s3transfer/__init__.py
index de62a74eba5..a15bac0cb9f 100644
--- a/contrib/python/s3transfer/py3/s3transfer/__init__.py
+++ b/contrib/python/s3transfer/py3/s3transfer/__init__.py
@@ -145,7 +145,7 @@ import s3transfer.compat
from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError
__author__ = 'Amazon Web Services'
-__version__ = '0.11.0'
+__version__ = '0.11.3'
class NullHandler(logging.Handler):
diff --git a/contrib/python/s3transfer/py3/s3transfer/crt.py b/contrib/python/s3transfer/py3/s3transfer/crt.py
index 37b041d8a96..b11c368261c 100644
--- a/contrib/python/s3transfer/py3/s3transfer/crt.py
+++ b/contrib/python/s3transfer/py3/s3transfer/crt.py
@@ -895,14 +895,22 @@ class S3ClientArgsCreator:
) and accesspoint_arn_details['region'] == "":
# Configure our region to `*` to propogate in `x-amz-region-set`
# for multi-region support in MRAP accesspoints.
+ # use_double_uri_encode and should_normalize_uri_path are defaulted to be True
+ # But SDK already encoded the URI, and it's for S3, so set both to False
make_request_args['signing_config'] = AwsSigningConfig(
algorithm=AwsSigningAlgorithm.V4_ASYMMETRIC,
region="*",
+ use_double_uri_encode=False,
+ should_normalize_uri_path=False,
)
call_args.bucket = accesspoint_arn_details['resource_name']
elif is_s3express_bucket(call_args.bucket):
+ # use_double_uri_encode and should_normalize_uri_path are defaulted to be True
+ # But SDK already encoded the URI, and it's for S3, so set both to False
make_request_args['signing_config'] = AwsSigningConfig(
- algorithm=AwsSigningAlgorithm.V4_S3EXPRESS
+ algorithm=AwsSigningAlgorithm.V4_S3EXPRESS,
+ use_double_uri_encode=False,
+ should_normalize_uri_path=False,
)
return make_request_args
diff --git a/contrib/python/s3transfer/py3/s3transfer/manager.py b/contrib/python/s3transfer/py3/s3transfer/manager.py
index 23b6f305ef0..8e1cdf068bb 100644
--- a/contrib/python/s3transfer/py3/s3transfer/manager.py
+++ b/contrib/python/s3transfer/py3/s3transfer/manager.py
@@ -519,7 +519,11 @@ class TransferManager:
)
def _add_operation_defaults(self, extra_args):
- set_default_checksum_algorithm(extra_args)
+ if (
+ self.client.meta.config.request_checksum_calculation
+ == "when_supported"
+ ):
+ set_default_checksum_algorithm(extra_args)
def _submit_transfer(
self, call_args, submission_task_cls, extra_main_kwargs=None
diff --git a/contrib/python/s3transfer/py3/tests/functional/test_crt.py b/contrib/python/s3transfer/py3/tests/functional/test_crt.py
index 5ef25144e85..98e3fffbe7a 100644
--- a/contrib/python/s3transfer/py3/tests/functional/test_crt.py
+++ b/contrib/python/s3transfer/py3/tests/functional/test_crt.py
@@ -163,6 +163,12 @@ class TestCRTTransferManager(unittest.TestCase):
make_request_kwargs['signing_config'].algorithm,
awscrt.auth.AwsSigningAlgorithm.V4_S3EXPRESS,
)
+ self.assertFalse(
+ make_request_kwargs['signing_config'].use_double_uri_encode,
+ )
+ self.assertFalse(
+ make_request_kwargs['signing_config'].should_normalize_uri_path,
+ )
def _assert_expected_mrap_request(
self, make_request_kwargs, expected_http_method='GET'
@@ -179,6 +185,12 @@ class TestCRTTransferManager(unittest.TestCase):
awscrt.auth.AwsSigningAlgorithm.V4_ASYMMETRIC,
)
self.assertEqual(make_request_kwargs['signing_config'].region, "*")
+ self.assertFalse(
+ make_request_kwargs['signing_config'].use_double_uri_encode,
+ )
+ self.assertFalse(
+ make_request_kwargs['signing_config'].should_normalize_uri_path,
+ )
def _assert_subscribers_called(self, expected_future=None):
self.assertTrue(self.record_subscriber.on_queued_called)
diff --git a/contrib/python/s3transfer/py3/tests/functional/test_upload.py b/contrib/python/s3transfer/py3/tests/functional/test_upload.py
index 00ac284f62f..efbc55af06c 100644
--- a/contrib/python/s3transfer/py3/tests/functional/test_upload.py
+++ b/contrib/python/s3transfer/py3/tests/functional/test_upload.py
@@ -19,6 +19,7 @@ from io import BytesIO
from botocore.awsrequest import AWSRequest
from botocore.client import Config
from botocore.exceptions import ClientError
+from botocore.httpchecksum import DEFAULT_CHECKSUM_ALGORITHM
from botocore.stub import ANY
from s3transfer.manager import TransferConfig, TransferManager
@@ -143,7 +144,7 @@ class TestNonMultipartUpload(BaseUploadTest):
__test__ = True
def add_put_object_response_with_default_expected_params(
- self, extra_expected_params=None, bucket=None
+ self, extra_expected_params=None, bucket=None, include_checksum=True
):
if bucket is None:
bucket = self.bucket
@@ -152,8 +153,9 @@ class TestNonMultipartUpload(BaseUploadTest):
'Body': ANY,
'Bucket': bucket,
'Key': self.key,
- 'ChecksumAlgorithm': 'CRC32',
}
+ if include_checksum:
+ expected_params["ChecksumAlgorithm"] = DEFAULT_CHECKSUM_ALGORITHM
if extra_expected_params:
expected_params.update(extra_expected_params)
upload_response = self.create_stubbed_responses()[0]
@@ -187,6 +189,46 @@ class TestNonMultipartUpload(BaseUploadTest):
self.assert_expected_client_calls_were_correct()
self.assert_put_object_body_was_correct()
+ def test_upload_with_default_checksum_when_supported(self):
+ # Reset client to configure `request_checksum_calculation` to "when_supported".
+ self.reset_stubber_with_new_client(
+ {'config': Config(request_checksum_calculation="when_supported")}
+ )
+ self.client.meta.events.register(
+ 'before-parameter-build.s3.*', self.collect_body
+ )
+ self._manager = TransferManager(self.client, self.config)
+
+ self.add_put_object_response_with_default_expected_params(
+ include_checksum=True
+ )
+ future = self.manager.upload(
+ self.filename, self.bucket, self.key, self.extra_args
+ )
+ future.result()
+ self.assert_expected_client_calls_were_correct()
+ self.assert_put_object_body_was_correct()
+
+ def test_upload_with_default_checksum_when_required(self):
+ # Reset client to configure `request_checksum_calculation` to "when_required".
+ self.reset_stubber_with_new_client(
+ {'config': Config(request_checksum_calculation="when_required")}
+ )
+ self.client.meta.events.register(
+ 'before-parameter-build.s3.*', self.collect_body
+ )
+ self._manager = TransferManager(self.client, self.config)
+
+ self.add_put_object_response_with_default_expected_params(
+ include_checksum=False
+ )
+ future = self.manager.upload(
+ self.filename, self.bucket, self.key, self.extra_args
+ )
+ future.result()
+ self.assert_expected_client_calls_were_correct()
+ self.assert_put_object_body_was_correct()
+
def test_upload_with_s3express_default_checksum(self):
s3express_bucket = "mytestbucket--usw2-az6--x-s3"
self.assertFalse("ChecksumAlgorithm" in self.extra_args)
@@ -370,9 +412,7 @@ class TestMultipartUpload(BaseUploadTest):
self.assertEqual(self.sent_bodies, expected_contents)
def add_create_multipart_response_with_default_expected_params(
- self,
- extra_expected_params=None,
- bucket=None,
+ self, extra_expected_params=None, bucket=None, include_checksum=True
):
if bucket is None:
bucket = self.bucket
@@ -380,8 +420,9 @@ class TestMultipartUpload(BaseUploadTest):
expected_params = {
'Bucket': bucket,
'Key': self.key,
- 'ChecksumAlgorithm': 'CRC32',
}
+ if include_checksum:
+ expected_params["ChecksumAlgorithm"] = DEFAULT_CHECKSUM_ALGORITHM
if extra_expected_params:
expected_params.update(extra_expected_params)
response = self.create_stubbed_responses()[0]
@@ -389,9 +430,7 @@ class TestMultipartUpload(BaseUploadTest):
self.stubber.add_response(**response)
def add_upload_part_responses_with_default_expected_params(
- self,
- extra_expected_params=None,
- bucket=None,
+ self, extra_expected_params=None, bucket=None, include_checksum=True
):
if bucket is None:
bucket = self.bucket
@@ -406,50 +445,48 @@ class TestMultipartUpload(BaseUploadTest):
'UploadId': self.multipart_id,
'Body': ANY,
'PartNumber': i + 1,
- 'ChecksumAlgorithm': 'CRC32',
}
+ if include_checksum:
+ expected_params["ChecksumAlgorithm"] = (
+ DEFAULT_CHECKSUM_ALGORITHM
+ )
if extra_expected_params:
expected_params.update(extra_expected_params)
- name = expected_params['ChecksumAlgorithm']
- checksum_member = f'Checksum{name.upper()}'
- response = upload_part_response['service_response']
- response[checksum_member] = f'sum{i+1}=='
+ # If ChecksumAlgorithm is in expected parameters, add checksum to the response
+ checksum_algorithm = expected_params.get('ChecksumAlgorithm')
+ if checksum_algorithm:
+ checksum_member = f'Checksum{checksum_algorithm.upper()}'
+ response = upload_part_response['service_response']
+ response[checksum_member] = f'sum{i+1}=='
upload_part_response['expected_params'] = expected_params
self.stubber.add_response(**upload_part_response)
def add_complete_multipart_response_with_default_expected_params(
- self,
- extra_expected_params=None,
- bucket=None,
+ self, extra_expected_params=None, bucket=None, include_checksum=True
):
if bucket is None:
bucket = self.bucket
+ num_parts = 3
+ parts = []
+ for part_num in range(1, num_parts + 1):
+ part = {
+ 'ETag': f'etag-{part_num}',
+ 'PartNumber': part_num,
+ }
+ if include_checksum:
+ part[f"Checksum{DEFAULT_CHECKSUM_ALGORITHM}"] = (
+ f"sum{part_num}=="
+ )
+ parts.append(part)
+
expected_params = {
'Bucket': bucket,
'Key': self.key,
'UploadId': self.multipart_id,
- 'MultipartUpload': {
- 'Parts': [
- {
- 'ETag': 'etag-1',
- 'PartNumber': 1,
- 'ChecksumCRC32': 'sum1==',
- },
- {
- 'ETag': 'etag-2',
- 'PartNumber': 2,
- 'ChecksumCRC32': 'sum2==',
- },
- {
- 'ETag': 'etag-3',
- 'PartNumber': 3,
- 'ChecksumCRC32': 'sum3==',
- },
- ]
- },
+ 'MultipartUpload': {'Parts': parts},
}
if extra_expected_params:
expected_params.update(extra_expected_params)
@@ -761,3 +798,53 @@ class TestMultipartUpload(BaseUploadTest):
)
future.result()
self.assert_expected_client_calls_were_correct()
+
+ def test_multipart_upload_with_default_checksum_when_supported(self):
+ # Reset client to configure `request_checksum_calculation` to "when_supported".
+ self.reset_stubber_with_new_client(
+ {'config': Config(request_checksum_calculation="when_supported")}
+ )
+ self.client.meta.events.register(
+ 'before-parameter-build.s3.*', self.collect_body
+ )
+ self._manager = TransferManager(self.client, self.config)
+
+ self.add_create_multipart_response_with_default_expected_params(
+ include_checksum=True
+ )
+ self.add_upload_part_responses_with_default_expected_params(
+ include_checksum=True
+ )
+ self.add_complete_multipart_response_with_default_expected_params()
+
+ future = self.manager.upload(
+ self.filename, self.bucket, self.key, self.extra_args
+ )
+ future.result()
+ self.assert_expected_client_calls_were_correct()
+
+ def test_multipart_upload_with_default_checksum_when_required(self):
+ # Reset client to configure `request_checksum_calculation` to "when_required".
+ self.reset_stubber_with_new_client(
+ {'config': Config(request_checksum_calculation="when_required")}
+ )
+ self.client.meta.events.register(
+ 'before-parameter-build.s3.*', self.collect_body
+ )
+ self._manager = TransferManager(self.client, self.config)
+
+ self.add_create_multipart_response_with_default_expected_params(
+ include_checksum=False
+ )
+ self.add_upload_part_responses_with_default_expected_params(
+ include_checksum=False
+ )
+ self.add_complete_multipart_response_with_default_expected_params(
+ include_checksum=False
+ )
+
+ future = self.manager.upload(
+ self.filename, self.bucket, self.key, self.extra_args
+ )
+ future.result()
+ self.assert_expected_client_calls_were_correct()
diff --git a/contrib/python/s3transfer/py3/ya.make b/contrib/python/s3transfer/py3/ya.make
index 57776c6f7ce..7c6a4458dcf 100644
--- a/contrib/python/s3transfer/py3/ya.make
+++ b/contrib/python/s3transfer/py3/ya.make
@@ -2,7 +2,7 @@
PY3_LIBRARY()
-VERSION(0.11.0)
+VERSION(0.11.3)
LICENSE(Apache-2.0)
diff --git a/contrib/python/scramp/.dist-info/METADATA b/contrib/python/scramp/.dist-info/METADATA
index d3b2cda4342..eb6b3e790f8 100644
--- a/contrib/python/scramp/.dist-info/METADATA
+++ b/contrib/python/scramp/.dist-info/METADATA
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: scramp
-Version: 1.4.9
+Version: 1.4.10
Summary: An implementation of the SCRAM protocol.
Project-URL: Homepage, https://codeberg.org/tlocke/scramp
Author: The Contributors
@@ -54,6 +54,7 @@ Scramp supports the following mechanisms:
* [OpenSSF Scorecard](#openssf-scorecard)
* [Doing A Release Of Scramp](#doing-a-release-of-scramp)
* [Release Notes](#release-notes)
+ * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27)
* [Version 1.4.9, 2026-06-19](#version-149-2026-06-19)
* [Version 1.4.8, 2026-01-06](#version-148-2026-01-06)
* [Version 1.4.7, 2026-01-04](#version-147-2026-01-04)
@@ -384,6 +385,15 @@ Run `tox` to make sure all tests pass, then update the release notes, then do:
## Release Notes
+### Version 1.4.10, 2026-06-27
+
+- Implement max and min guards for iteration count.
+- The xor() function should fail if the two values are of unequal lengths
+- Handle the case of an error in first server message
+- Fix formatting in channel binding error message
+- SHA3-512 didn't work, but it should do now
+
+
### Version 1.4.9, 2026-06-19
- Fix typo in error message.
diff --git a/contrib/python/scramp/README.md b/contrib/python/scramp/README.md
index 1ccdd745c56..652f8f590d0 100644
--- a/contrib/python/scramp/README.md
+++ b/contrib/python/scramp/README.md
@@ -33,6 +33,7 @@ Scramp supports the following mechanisms:
* [OpenSSF Scorecard](#openssf-scorecard)
* [Doing A Release Of Scramp](#doing-a-release-of-scramp)
* [Release Notes](#release-notes)
+ * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27)
* [Version 1.4.9, 2026-06-19](#version-149-2026-06-19)
* [Version 1.4.8, 2026-01-06](#version-148-2026-01-06)
* [Version 1.4.7, 2026-01-04](#version-147-2026-01-04)
@@ -363,6 +364,15 @@ Run `tox` to make sure all tests pass, then update the release notes, then do:
## Release Notes
+### Version 1.4.10, 2026-06-27
+
+- Implement max and min guards for iteration count.
+- The xor() function should fail if the two values are of unequal lengths
+- Handle the case of an error in first server message
+- Fix formatting in channel binding error message
+- SHA3-512 didn't work, but it should do now
+
+
### Version 1.4.9, 2026-06-19
- Fix typo in error message.
diff --git a/contrib/python/scramp/scramp/core.py b/contrib/python/scramp/scramp/core.py
index 7bfd362914a..7ab1a5703a7 100644
--- a/contrib/python/scramp/scramp/core.py
+++ b/contrib/python/scramp/scramp/core.py
@@ -77,6 +77,8 @@ CHANNEL_TYPES = (
"tls-unique-for-telnet",
)
+MAX_ITERATION_COUNT = 10_000_000 # DoS guard
+
def _make_cb_data(name, ssl_socket):
if name == "tls-unique":
@@ -137,6 +139,10 @@ class ScramMechanism:
def make_auth_info(self, password, iteration_count=None, salt=None):
if iteration_count is None:
iteration_count = self.iteration_count
+ if iteration_count < self.iteration_count:
+ raise ScramException(
+ f"The iteration count must be at least {self.iteration_count}"
+ )
salt, stored_key, server_key = _make_auth_info(
self.hf, password, iteration_count, salt=salt
)
@@ -175,8 +181,8 @@ def _validate_channel_binding(channel_binding):
channel_type, channel_data = channel_binding
if channel_type not in CHANNEL_TYPES:
raise ScramException(
- "The channel_binding parameter must either be None or a tuple with the "
- "first element a str specifying one of the channel types {CHANNEL_TYPES}."
+ f"The channel_binding parameter must either be None or a tuple with the "
+ f"first element a str specifying one of the channel types {CHANNEL_TYPES}."
)
if not isinstance(channel_data, bytes):
@@ -207,6 +213,7 @@ class ScramClient:
mech = sorted(mechs, key=attrgetter("strength"))[-1]
self.hf, self.use_binding = mech.hf, mech.use_binding
self.mechanism_name = mech.name
+ self.iterations = mech.iteration_count
self.c_nonce = _make_nonce() if c_nonce is None else c_nonce
self.username = username
@@ -229,7 +236,7 @@ class ScramClient:
self._set_stage(ClientStage.set_server_first)
self.server_first = message
self.nonce, self.salt, self.iterations = _set_server_first(
- message, self.c_nonce
+ message, self.c_nonce, self.iterations
)
def get_client_final(self):
@@ -351,8 +358,7 @@ def _make_auth_message(client_first_bare, server_first, client_final_without_pro
def _make_salted_password(hf, password, salt, iterations):
- hash_name = hf.__name__.split("_")[-1]
- return hashlib.pbkdf2_hmac(hash_name, uenc(saslprep(password)), salt, iterations)
+ return hashlib.pbkdf2_hmac(hf().name, uenc(saslprep(password)), salt, iterations)
def _c_key_stored_key_s_key(hf, salted_password):
@@ -518,14 +524,19 @@ def _get_server_first(nonce, salt, iterations):
return ",".join((f"r={nonce}", f"s={salt}", f"i={iterations}"))
-def _set_server_first(server_first, c_nonce):
- msg = _parse_message(server_first, "server first", {"r", "s", "i"})
+def _set_server_first(server_first, c_nonce, min_iteration_count):
+ msg = _parse_message(server_first, "server first", {"r", "s", "i"}, {"e"})
if "e" in msg:
raise ScramException(f"The server returned the error: {msg['e']}")
nonce = msg["r"]
salt = msg["s"]
iterations = int(msg["i"])
+ if not (min_iteration_count <= iterations <= MAX_ITERATION_COUNT):
+ raise ScramException(
+ f"Server iteration count must be between {min_iteration_count} "
+ f"and {MAX_ITERATION_COUNT} inclusive"
+ )
if not nonce.startswith(c_nonce):
raise ScramException("Client nonce doesn't match.", SERVER_ERROR_OTHER_ERROR)
diff --git a/contrib/python/scramp/scramp/utils.py b/contrib/python/scramp/scramp/utils.py
index da864d2fe2f..da991644090 100644
--- a/contrib/python/scramp/scramp/utils.py
+++ b/contrib/python/scramp/scramp/utils.py
@@ -14,7 +14,7 @@ def h(hf, msg):
def xor(bytes1, bytes2):
- return bytes(a ^ b for a, b in zip(bytes1, bytes2))
+ return bytes(a ^ b for a, b in zip(bytes1, bytes2, strict=True))
def b64enc(binary):
diff --git a/contrib/python/scramp/ya.make b/contrib/python/scramp/ya.make
index 9453f63f0af..f5d6699f70a 100644
--- a/contrib/python/scramp/ya.make
+++ b/contrib/python/scramp/ya.make
@@ -2,7 +2,7 @@
PY3_LIBRARY()
-VERSION(1.4.9)
+VERSION(1.4.10)
LICENSE(MIT-0)
diff --git a/library/cpp/yt/coding/bit_io-inl.h b/library/cpp/yt/coding/bit_io-inl.h
new file mode 100644
index 00000000000..89d69be6251
--- /dev/null
+++ b/library/cpp/yt/coding/bit_io-inl.h
@@ -0,0 +1,69 @@
+#ifndef BIT_IO_INL_H_
+#error "Direct inclusion of this file is not allowed, include bit_io.h"
+// For the sake of sane code completion.
+#include "bit_io.h"
+#endif
+
+#include <util/system/compiler.h>
+#include <util/system/unaligned_mem.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline TBitWriter::TBitWriter(char* ptr)
+ : Ptr_(ptr)
+{ }
+
+Y_FORCE_INLINE void TBitWriter::WriteBits(ui32 value, int width)
+{
+ // Flush a whole 32-bit word at once (a single unaligned store) instead of
+ // looping over individual bytes with a data-dependent trip count.
+ Accumulator_ = (Accumulator_ << width) | value;
+ BitCount_ += width;
+ if (BitCount_ >= 32) {
+ BitCount_ -= 32;
+ ui32 word = __builtin_bswap32(static_cast<ui32>(Accumulator_ >> BitCount_));
+ WriteUnaligned<ui32>(Ptr_, word);
+ Ptr_ += sizeof(word);
+ }
+}
+
+inline char* TBitWriter::Finish()
+{
+ while (BitCount_ >= 8) {
+ BitCount_ -= 8;
+ *Ptr_++ = static_cast<char>((Accumulator_ >> BitCount_) & 0xff);
+ }
+ if (BitCount_ > 0) {
+ *Ptr_++ = static_cast<char>((Accumulator_ << (8 - BitCount_)) & 0xff);
+ BitCount_ = 0;
+ }
+ Accumulator_ = 0;
+ return Ptr_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline TBitReader::TBitReader(const char* ptr)
+ : Ptr_(reinterpret_cast<const ui8*>(ptr))
+{ }
+
+Y_FORCE_INLINE ui32 TBitReader::ReadBits(int width)
+{
+ while (BitCount_ < width) {
+ Accumulator_ = (Accumulator_ << 8) | *Ptr_++;
+ BitCount_ += 8;
+ }
+ BitCount_ -= width;
+ return static_cast<ui32>((Accumulator_ >> BitCount_) & ((1ull << width) - 1));
+}
+
+inline const char* TBitReader::Finish()
+{
+ return reinterpret_cast<const char*>(Ptr_);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/coding/bit_io.h b/library/cpp/yt/coding/bit_io.h
new file mode 100644
index 00000000000..18900d69f85
--- /dev/null
+++ b/library/cpp/yt/coding/bit_io.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include <util/system/types.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! MSB-first bit writer over a caller-owned buffer.
+/*!
+ * Bits are flushed to the buffer in 4-byte chunks, so the buffer must have room
+ * for up to 3 bytes beyond the last logically-written byte before #Finish is
+ * called.
+ */
+class TBitWriter
+{
+public:
+ explicit TBitWriter(char* ptr);
+
+ //! Appends the #width low bits of #value. Requires 0 <= #width <= 32 and
+ //! #value < 2^#width (for #width == 32 any value is accepted).
+ void WriteBits(ui32 value, int width);
+
+ //! Pads the last partial byte with zero low bits and returns the
+ //! one-past-end pointer.
+ char* Finish();
+
+private:
+ char* Ptr_;
+ ui64 Accumulator_ = 0;
+ int BitCount_ = 0;
+};
+
+//! MSB-first bit reader.
+/*!
+ * Reads bits written by #TBitWriter. Assumes up to 8 bytes past the logical end
+ * of the stream are safe to read.
+ */
+class TBitReader
+{
+public:
+ explicit TBitReader(const char* ptr);
+
+ //! Reads and returns #width bits (0 <= #width <= 32).
+ ui32 ReadBits(int width);
+
+ //! Returns the one-past-end pointer; the (< 8) buffered sub-byte bits, which
+ //! are the writer's zero padding, are dropped.
+ const char* Finish();
+
+private:
+ const ui8* Ptr_;
+ ui64 Accumulator_ = 0;
+ int BitCount_ = 0;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
+#define BIT_IO_INL_H_
+#include "bit_io-inl.h"
+#undef BIT_IO_INL_H_
diff --git a/library/cpp/yt/coding/interpolative-inl.h b/library/cpp/yt/coding/interpolative-inl.h
new file mode 100644
index 00000000000..5eadfdd483e
--- /dev/null
+++ b/library/cpp/yt/coding/interpolative-inl.h
@@ -0,0 +1,182 @@
+#ifndef INTERPOLATIVE_INL_H_
+#error "Direct inclusion of this file is not allowed, include interpolative.h"
+// For the sake of sane code completion.
+#include "interpolative.h"
+#endif
+
+#include "bit_io.h"
+
+#include <library/cpp/yt/assert/assert.h>
+
+#include <library/cpp/yt/memory/range.h>
+
+#include <util/system/compiler.h>
+
+#include <array>
+#include <bit>
+#include <concepts>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NInterpolativeCodingDetail {
+
+// Truncated-binary (minimal) code for a value in [0, rangeSize): the first
+// Cutoff values take LowWidth bits, the rest one more.
+struct TTruncatedBinaryParams
+{
+ int LowWidth; // floor(log2(rangeSize))
+ ui32 Cutoff; // 2^(LowWidth + 1) - rangeSize
+};
+
+Y_FORCE_INLINE TTruncatedBinaryParams GetTruncatedBinaryParams(ui32 rangeSize)
+{
+ int lowWidth = std::bit_width(rangeSize) - 1;
+ ui32 cutoff = (2u << lowWidth) - rangeSize;
+ return {lowWidth, cutoff};
+}
+
+// Writes #value in [0, #rangeSize) with the entropy-optimal integer code for a
+// uniform value. rangeSize == 1 yields lowWidth 0 / cutoff 1 and emits a
+// zero-width code (a no-op), so the singleton case needs no branch.
+Y_FORCE_INLINE void WriteTruncatedBinary(TBitWriter* writer, ui32 value, ui32 rangeSize)
+{
+ auto [lowWidth, cutoff] = GetTruncatedBinaryParams(rangeSize);
+ // Branchless: the (value >= cutoff) predicate would mispredict on nearly
+ // every element, so fold it into the emitted codeword and width instead.
+ ui32 isLong = value >= cutoff ? 1 : 0;
+ writer->WriteBits(value + (cutoff & (0u - isLong)), lowWidth + static_cast<int>(isLong));
+}
+
+Y_FORCE_INLINE ui32 ReadTruncatedBinary(TBitReader* reader, ui32 rangeSize)
+{
+ auto [lowWidth, cutoff] = GetTruncatedBinaryParams(rangeSize);
+ ui32 high = reader->ReadBits(lowWidth);
+ if (high < cutoff) {
+ return high;
+ }
+ ui32 low = reader->ReadBits(1);
+ return ((high << 1) | low) - cutoff;
+}
+
+// A subrange [BeginIndex, EndIndex) of the value array together with the
+// inclusive value bounds [Lo, Hi] that its elements must fall in. The median
+// values[m] is the only array element touched per node, which matters because
+// the traversal walks a large sequence in a cache-unfriendly tree order.
+struct TInterpolativeFrame
+{
+ int BeginIndex;
+ int EndIndex;
+ ui32 Lo;
+ ui32 Hi;
+};
+
+} // namespace NInterpolativeCodingDetail
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline size_t GetInterpolativeMaxByteSize(int count, ui32 lo, ui32 hi)
+{
+ // Each element is coded in at most ceil(log2(hi - lo + 1)) bits; the trailing
+ // word covers the writer's 4-byte flush store.
+ int maxBitWidth = std::bit_width(hi - lo);
+ return (static_cast<size_t>(count) * maxBitWidth + 7) / 8 + sizeof(ui32);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Both traversals descend the left child in place and stack only the (non-empty)
+// right child, so the stack sees ~count/2 pushes instead of ~count. Left-first
+// order matches between encoder and decoder.
+template <std::unsigned_integral T>
+void InterpolativeEncode(TBitWriter* writer, TRange<T> values, ui32 lo, ui32 hi)
+{
+ using namespace NInterpolativeCodingDetail;
+
+ int count = std::ssize(values);
+ if (count == 0) {
+ return;
+ }
+
+ // Right children stack up along the leftmost path => depth <= ceil(log2(count)).
+ std::array<TInterpolativeFrame, 48> stack;
+ int top = 0;
+ int beginIndex = 0;
+ int endIndex = count;
+ ui32 l = lo;
+ ui32 h = hi;
+ for (;;) {
+ // beginIndex is invariant while descending left; only endIndex/l/h change.
+ while (beginIndex < endIndex) {
+ int half = (endIndex - beginIndex) / 2;
+ int m = beginIndex + half;
+ // The next descent step reads values[beginIndex + half/2]; prefetch it
+ // to hide the cache-scattered tree walk on large sequences.
+ Y_PREFETCH_READ(values.data() + beginIndex + (half >> 1), 0);
+ ui32 value = static_cast<ui32>(values[m]);
+ // rangeSize = (upperBound - lowerBound + 1) simplifies to this;
+ // lowerBound = l + half.
+ ui32 rangeSize = (h - l) - static_cast<ui32>(endIndex - beginIndex) + 2;
+ WriteTruncatedBinary(writer, value - l - static_cast<ui32>(half), rangeSize);
+ if (m + 1 < endIndex) {
+ YT_ASSERT(top < std::ssize(stack));
+ stack[top++] = {m + 1, endIndex, value + 1, h};
+ }
+ endIndex = m;
+ h = value - 1;
+ }
+ if (top == 0) {
+ break;
+ }
+ auto f = stack[--top];
+ beginIndex = f.BeginIndex;
+ endIndex = f.EndIndex;
+ l = f.Lo;
+ h = f.Hi;
+ }
+}
+
+template <std::unsigned_integral T>
+void InterpolativeDecode(TBitReader* reader, TMutableRange<T> values, ui32 lo, ui32 hi)
+{
+ using namespace NInterpolativeCodingDetail;
+
+ int count = std::ssize(values);
+ if (count == 0) {
+ return;
+ }
+
+ std::array<TInterpolativeFrame, 48> stack;
+ int top = 0;
+ int beginIndex = 0;
+ int endIndex = count;
+ ui32 l = lo;
+ ui32 h = hi;
+ for (;;) {
+ while (beginIndex < endIndex) {
+ int half = (endIndex - beginIndex) / 2;
+ int m = beginIndex + half;
+ ui32 rangeSize = (h - l) - static_cast<ui32>(endIndex - beginIndex) + 2;
+ ui32 value = l + static_cast<ui32>(half) + ReadTruncatedBinary(reader, rangeSize);
+ values[m] = static_cast<T>(value);
+ if (m + 1 < endIndex) {
+ stack[top++] = {m + 1, endIndex, value + 1, h};
+ }
+ endIndex = m;
+ h = value - 1;
+ }
+ if (top == 0) {
+ break;
+ }
+ auto f = stack[--top];
+ beginIndex = f.BeginIndex;
+ endIndex = f.EndIndex;
+ l = f.Lo;
+ h = f.Hi;
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yt/coding/interpolative.h b/library/cpp/yt/coding/interpolative.h
new file mode 100644
index 00000000000..88e415e754c
--- /dev/null
+++ b/library/cpp/yt/coding/interpolative.h
@@ -0,0 +1,52 @@
+#pragma once
+
+#include "bit_io.h"
+
+#include <library/cpp/yt/memory/range.h>
+
+#include <util/system/types.h>
+
+#include <concepts>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+//
+// Binary interpolative coding for a sorted, strictly increasing sequence of
+// integers drawn from a known range [lo, hi]. The value domain is 32-bit: lo, hi
+// and hence every value fit in ui32, independent of the element type T (which is
+// merely the container's width).
+//
+// It recursively encodes the median element within the range implied by its
+// position and its already-coded neighbors, so clustered sequences compress far
+// below a flat log2 per element and no per-element headers are needed. Each
+// element is stored with a truncated-binary (minimal) code, which spends the
+// fractional part of log2(range) instead of rounding every element up to a whole
+// bit.
+//
+// The bit stream is MSB-first (see bit_io.h).
+//
+////////////////////////////////////////////////////////////////////////////////
+
+//! Encodes #values, which must be strictly increasing and all within [#lo, #hi],
+//! with binary interpolative coding. An empty range emits nothing; the length is
+//! not stored and must be conveyed out of band (e.g. as a varint prefix).
+template <std::unsigned_integral T>
+void InterpolativeEncode(TBitWriter* writer, TRange<T> values, ui32 lo, ui32 hi);
+
+//! Decodes a sequence written by #InterpolativeEncode into #values, whose size
+//! must equal the encoded element count. #lo and #hi must match the encoder.
+template <std::unsigned_integral T>
+void InterpolativeDecode(TBitReader* reader, TMutableRange<T> values, ui32 lo, ui32 hi);
+
+//! An upper bound on the buffer size #InterpolativeEncode needs to encode #count
+//! values over [#lo, #hi], including the slack the writer requires.
+size_t GetInterpolativeMaxByteSize(int count, ui32 lo, ui32 hi);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
+
+#define INTERPOLATIVE_INL_H_
+#include "interpolative-inl.h"
+#undef INTERPOLATIVE_INL_H_
diff --git a/library/cpp/yt/coding/unittests/bit_io_ut.cpp b/library/cpp/yt/coding/unittests/bit_io_ut.cpp
new file mode 100644
index 00000000000..5d61b166f0a
--- /dev/null
+++ b/library/cpp/yt/coding/unittests/bit_io_ut.cpp
@@ -0,0 +1,69 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/coding/bit_io.h>
+
+#include <utility>
+#include <vector>
+
+namespace NYT {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(TBitIOTest, RoundTrip)
+{
+ std::vector<std::pair<ui32, int>> items = {
+ {0, 0}, {1, 1}, {0, 1}, {5, 3}, {0, 8}, {255, 8}, {12345, 14},
+ {0, 32}, {0xffffffffu, 32}, {0x12345, 17}, {7, 3}, {1u << 30, 31},
+ };
+ std::vector<char> buffer(256, 0);
+
+ TBitWriter writer(buffer.data());
+ for (auto [value, width] : items) {
+ writer.WriteBits(value, width);
+ }
+ writer.Finish();
+
+ TBitReader reader(buffer.data());
+ for (auto [value, width] : items) {
+ EXPECT_EQ(reader.ReadBits(width), value) << "width=" << width;
+ }
+}
+
+TEST(TBitIOTest, ZeroWidthIsNoop)
+{
+ std::vector<char> buffer(16, 0);
+ TBitWriter writer(buffer.data());
+ writer.WriteBits(0, 0);
+ writer.WriteBits(1, 1);
+ writer.WriteBits(0, 0);
+ char* end = writer.Finish();
+ EXPECT_EQ(end - buffer.data(), 1); // a single bit occupies one byte
+
+ TBitReader reader(buffer.data());
+ EXPECT_EQ(reader.ReadBits(0), 0u);
+ EXPECT_EQ(reader.ReadBits(1), 1u);
+ EXPECT_EQ(reader.ReadBits(0), 0u);
+}
+
+TEST(TBitIOTest, FlushBoundary)
+{
+ // Many 17-bit writes repeatedly cross the internal 32-bit flush boundary.
+ constexpr int Count = 1000;
+ std::vector<char> buffer(Count * 3 + 16, 0);
+ TBitWriter writer(buffer.data());
+ for (int i = 0; i < Count; ++i) {
+ writer.WriteBits(static_cast<ui32>(i) & 0x1ffff, 17);
+ }
+ writer.Finish();
+
+ TBitReader reader(buffer.data());
+ for (int i = 0; i < Count; ++i) {
+ EXPECT_EQ(reader.ReadBits(17), static_cast<ui32>(i) & 0x1ffff) << "i=" << i;
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/library/cpp/yt/coding/unittests/interpolative_ut.cpp b/library/cpp/yt/coding/unittests/interpolative_ut.cpp
new file mode 100644
index 00000000000..a63b12f6eac
--- /dev/null
+++ b/library/cpp/yt/coding/unittests/interpolative_ut.cpp
@@ -0,0 +1,221 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/coding/interpolative.h>
+#include <library/cpp/yt/coding/varint.h>
+
+#include <algorithm>
+#include <bit>
+#include <numeric>
+#include <random>
+#include <set>
+#include <vector>
+
+namespace NYT {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+// Truncated binary
+
+TEST(TTruncatedBinaryTest, Exhaustive)
+{
+ for (ui32 rangeSize = 1; rangeSize <= 2050; ++rangeSize) {
+ for (ui32 value = 0; value < rangeSize; ++value) {
+ std::vector<char> buffer(16, 0);
+ TBitWriter writer(buffer.data());
+ NInterpolativeCodingDetail::WriteTruncatedBinary(&writer, value, rangeSize);
+ writer.Finish();
+
+ TBitReader reader(buffer.data());
+ EXPECT_EQ(NInterpolativeCodingDetail::ReadTruncatedBinary(&reader, rangeSize), value)
+ << "rangeSize=" << rangeSize << " value=" << value;
+ }
+ }
+}
+
+TEST(TTruncatedBinaryTest, MinimalLength)
+{
+ // Every codeword is floor(log2(rangeSize)) or ceil(log2(rangeSize)) bits.
+ for (ui32 rangeSize = 1; rangeSize <= 1000; ++rangeSize) {
+ int lowWidth = std::bit_width(rangeSize) - 1;
+ for (ui32 value = 0; value < rangeSize; ++value) {
+ std::vector<char> buffer(16, 0);
+ TBitWriter writer(buffer.data());
+ NInterpolativeCodingDetail::WriteTruncatedBinary(&writer, value, rangeSize);
+ char* end = writer.Finish();
+
+ TBitReader reader(buffer.data());
+ NInterpolativeCodingDetail::ReadTruncatedBinary(&reader, rangeSize);
+ const char* readEnd = reader.Finish();
+ i64 bytes = end - buffer.data();
+ // Read must consume no more bytes than were written.
+ EXPECT_LE(readEnd - buffer.data(), bytes);
+ EXPECT_LE(bytes, (lowWidth + 1 + 7) / 8 + 1);
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Interpolative coding
+
+template <class T>
+std::vector<char> Encode(const std::vector<T>& values, ui32 lo, ui32 hi)
+{
+ std::vector<char> buffer(values.size() * sizeof(ui32) + 16, 0);
+ TBitWriter writer(buffer.data());
+ InterpolativeEncode(&writer, TRange(values), lo, hi);
+ char* end = writer.Finish();
+ buffer.resize(end - buffer.data());
+ return buffer;
+}
+
+template <class T>
+std::vector<T> Decode(std::vector<char> buffer, int count, ui32 lo, ui32 hi)
+{
+ buffer.resize(buffer.size() + 8, 0); // reader may over-read up to 8 bytes
+ std::vector<T> values(count);
+ TBitReader reader(buffer.data());
+ InterpolativeDecode(&reader, TMutableRange(values), lo, hi);
+ return values;
+}
+
+template <class T>
+void ExpectRoundTrip(const std::vector<T>& values, ui32 lo, ui32 hi)
+{
+ auto decoded = Decode<T>(Encode(values, lo, hi), std::ssize(values), lo, hi);
+ EXPECT_EQ(decoded, values);
+}
+
+TEST(TInterpolativeCodingTest, Empty)
+{
+ ExpectRoundTrip<ui32>({}, 0, 100);
+}
+
+TEST(TInterpolativeCodingTest, Single)
+{
+ ExpectRoundTrip<ui32>({0}, 0, 0);
+ ExpectRoundTrip<ui32>({42}, 0, 100);
+ ExpectRoundTrip<ui32>({100}, 0, 100);
+}
+
+TEST(TInterpolativeCodingTest, SmallCases)
+{
+ ExpectRoundTrip<ui32>({3, 7}, 0, 10);
+ ExpectRoundTrip<ui32>({0, 1, 2}, 0, 2); // full, zero bits
+ ExpectRoundTrip<ui32>({0, 5, 10}, 0, 10); // boundaries present
+ ExpectRoundTrip<ui32>({1, 2, 3, 4, 5}, 0, 6);
+}
+
+TEST(TInterpolativeCodingTest, FullRange)
+{
+ // Every value present => every range collapses to a singleton (zero bits).
+ std::vector<ui32> values(500);
+ std::iota(values.begin(), values.end(), 7);
+ auto encoded = Encode(values, 7, 506);
+ EXPECT_TRUE(encoded.empty());
+ EXPECT_EQ(Decode<ui32>(encoded, 500, 7, 506), values);
+}
+
+TEST(TInterpolativeCodingTest, RandomRoundTrip)
+{
+ std::mt19937 rng(12345);
+ for (int iteration = 0; iteration < 500; ++iteration) {
+ ui32 universe = 1 + rng() % 200'000;
+ int count = std::min<ui32>(universe, 1 + rng() % 2000);
+ std::set<ui32> unique;
+ std::uniform_int_distribution<ui32> dist(0, universe - 1);
+ while (std::ssize(unique) < count) {
+ unique.insert(dist(rng));
+ }
+ std::vector<ui32> values(unique.begin(), unique.end());
+ ExpectRoundTrip<ui32>(values, 0, universe - 1);
+ }
+}
+
+TEST(TInterpolativeCodingTest, NonZeroLowerBound)
+{
+ std::mt19937 rng(999);
+ for (int iteration = 0; iteration < 200; ++iteration) {
+ ui32 lo = rng() % 100'000;
+ ui32 span = 1 + rng() % 100'000;
+ ui32 hi = lo + span;
+ int count = std::min<ui32>(span + 1, 1 + rng() % 500);
+ std::set<ui32> unique;
+ std::uniform_int_distribution<ui32> dist(lo, hi);
+ while (std::ssize(unique) < count) {
+ unique.insert(dist(rng));
+ }
+ std::vector<ui32> values(unique.begin(), unique.end());
+ ExpectRoundTrip<ui32>(values, lo, hi);
+ }
+}
+
+TEST(TInterpolativeCodingTest, Ui64Values)
+{
+ std::vector<ui64> values = {0, 1, 100, 1000, 50'000, 200'000};
+ ExpectRoundTrip<ui64>(values, 0, 200'000);
+}
+
+TEST(TInterpolativeCodingTest, MaxByteSize)
+{
+ std::mt19937 rng(555);
+ for (int iteration = 0; iteration < 300; ++iteration) {
+ ui32 universe = 1 + rng() % 200'000;
+ int count = std::min<ui32>(universe, 1 + rng() % 1000);
+ std::set<ui32> unique;
+ std::uniform_int_distribution<ui32> dist(0, universe - 1);
+ while (std::ssize(unique) < count) {
+ unique.insert(dist(rng));
+ }
+ std::vector<ui32> values(unique.begin(), unique.end());
+
+ size_t bound = GetInterpolativeMaxByteSize(count, 0, universe - 1);
+ std::vector<char> buffer(bound, 0);
+ TBitWriter writer(buffer.data());
+ InterpolativeEncode(&writer, TRange(values), 0, universe - 1);
+ EXPECT_LE(static_cast<size_t>(writer.Finish() - buffer.data()), bound);
+ }
+}
+
+TEST(TInterpolativeCodingTest, MultipleListsInOneBuffer)
+{
+ // Mirrors real usage: each list is prefixed with a varint length and is
+ // byte-aligned, so lists can be concatenated and read back in sequence.
+ std::mt19937 rng(7);
+ ui32 hi = 199999;
+ std::vector<std::vector<ui32>> lists;
+ for (int listIndex = 0; listIndex < 50; ++listIndex) {
+ int count = 1 + rng() % 300;
+ std::set<ui32> unique;
+ std::uniform_int_distribution<ui32> dist(0, hi);
+ while (std::ssize(unique) < count) {
+ unique.insert(dist(rng));
+ }
+ lists.emplace_back(unique.begin(), unique.end());
+ }
+
+ std::vector<char> buffer(200'000, 0);
+ char* ptr = buffer.data();
+ for (const auto& list : lists) {
+ ptr += WriteVarUint32(ptr, static_cast<ui32>(list.size()));
+ TBitWriter writer(ptr);
+ InterpolativeEncode(&writer, TRange(list), 0, hi);
+ ptr = writer.Finish();
+ }
+
+ const char* readPtr = buffer.data();
+ for (const auto& list : lists) {
+ ui32 count;
+ readPtr += ReadVarUint32(readPtr, &count);
+ ASSERT_EQ(count, list.size());
+ std::vector<ui32> decoded(count);
+ TBitReader reader(readPtr);
+ InterpolativeDecode(&reader, TMutableRange(decoded), 0, hi);
+ readPtr = reader.Finish();
+ EXPECT_EQ(decoded, list);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT
diff --git a/library/cpp/yt/coding/unittests/ya.make b/library/cpp/yt/coding/unittests/ya.make
index ab94ee8796a..945e63ab506 100644
--- a/library/cpp/yt/coding/unittests/ya.make
+++ b/library/cpp/yt/coding/unittests/ya.make
@@ -3,6 +3,8 @@ GTEST()
INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc)
SRCS(
+ bit_io_ut.cpp
+ interpolative_ut.cpp
zig_zag_ut.cpp
varint_ut.cpp
)
diff --git a/library/cpp/yt/coding/ya.make b/library/cpp/yt/coding/ya.make
index 639d94e755c..e5ea301cf90 100644
--- a/library/cpp/yt/coding/ya.make
+++ b/library/cpp/yt/coding/ya.make
@@ -6,7 +6,9 @@ SRCS(
)
PEERDIR(
+ library/cpp/yt/assert
library/cpp/yt/exception
+ library/cpp/yt/memory
)
END()
diff --git a/library/cpp/yt/coding/zig_zag-inl.h b/library/cpp/yt/coding/zig_zag-inl.h
index c611f7e1d46..9df9a2cc5f4 100644
--- a/library/cpp/yt/coding/zig_zag-inl.h
+++ b/library/cpp/yt/coding/zig_zag-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "zig_zag.h"
#endif
-#undef ZIG_ZAG_INL_H_
namespace NYT {
diff --git a/library/cpp/yt/compact_containers/compact_vector-inl.h b/library/cpp/yt/compact_containers/compact_vector-inl.h
index cec19960e19..1070e6b1c91 100644
--- a/library/cpp/yt/compact_containers/compact_vector-inl.h
+++ b/library/cpp/yt/compact_containers/compact_vector-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "compact_vector.h"
#endif
-#undef COMPACT_VECTOR_INL_H_
#include <library/cpp/yt/assert/assert.h>
diff --git a/library/cpp/yt/error/error.cpp b/library/cpp/yt/error/error.cpp
index d0a5489ab22..dc5aca7ccd2 100644
--- a/library/cpp/yt/error/error.cpp
+++ b/library/cpp/yt/error/error.cpp
@@ -5,6 +5,8 @@
#include <library/cpp/yt/error/error_attributes.h>
#include <library/cpp/yt/error/origin_attributes.h>
+#include <library/cpp/yt/memory/leaky_singleton.h>
+
#include <library/cpp/yt/string/string.h>
#include <library/cpp/yt/system/proc.h>
@@ -29,8 +31,20 @@ void FormatValue(TStringBuilderBase* builder, TErrorCode code, TStringBuf spec)
constexpr TStringBuf ErrorMessageTruncatedSuffix = "...<message truncated>";
-TError::TEnricher TError::Enricher_;
-TError::TFromExceptionEnricher TError::FromExceptionEnricher_;
+namespace {
+
+struct TEnricherStorage
+{
+ static TEnricherStorage* Get()
+ {
+ return LeakySingleton<TEnricherStorage>();
+ }
+
+ TError::TEnricher Enricher;
+ TError::TFromExceptionEnricher FromExceptionEnricher;
+};
+
+} // namespace
////////////////////////////////////////////////////////////////////////////////
@@ -636,11 +650,12 @@ void TError::RegisterEnricher(TEnricher enricher)
{
// NB: This daisy-chaining strategy is optimal when there's O(1) callbacks. Convert to a vector
// if the number grows.
- if (!Enricher_) {
- Enricher_ = std::move(enricher);
+ auto* storage = TEnricherStorage::Get();
+ if (!storage->Enricher) {
+ storage->Enricher = std::move(enricher);
return;
}
- Enricher_ = [first = std::move(Enricher_), second = std::move(enricher)] (TError* error) {
+ storage->Enricher = [first = std::move(storage->Enricher), second = std::move(enricher)] (TError* error) {
first(error);
second(error);
};
@@ -650,12 +665,13 @@ void TError::RegisterFromExceptionEnricher(TFromExceptionEnricher enricher)
{
// NB: This daisy-chaining strategy is optimal when there's O(1) callbacks. Convert to a vector
// if the number grows.
- if (!FromExceptionEnricher_) {
- FromExceptionEnricher_ = std::move(enricher);
+ auto* storage = TEnricherStorage::Get();
+ if (!storage->FromExceptionEnricher) {
+ storage->FromExceptionEnricher = std::move(enricher);
return;
}
- FromExceptionEnricher_ = [
- first = std::move(FromExceptionEnricher_),
+ storage->FromExceptionEnricher = [
+ first = std::move(storage->FromExceptionEnricher),
second = std::move(enricher)
] (TError* error, const std::exception& exception) {
first(error, exception);
@@ -676,15 +692,15 @@ void TError::MakeMutable()
void TError::Enrich()
{
- if (Enricher_) {
- Enricher_(this);
+ if (const auto& enricher = TEnricherStorage::Get()->Enricher) {
+ enricher(this);
}
}
void TError::EnrichFromException(const std::exception& exception)
{
- if (FromExceptionEnricher_) {
- FromExceptionEnricher_(this, exception);
+ if (const auto& enricher = TEnricherStorage::Get()->FromExceptionEnricher) {
+ enricher(this, exception);
}
}
diff --git a/library/cpp/yt/error/error.h b/library/cpp/yt/error/error.h
index b2d0abacfe8..13223af8d6c 100644
--- a/library/cpp/yt/error/error.h
+++ b/library/cpp/yt/error/error.h
@@ -253,9 +253,6 @@ private:
void EnrichFromException(const std::exception& exception);
friend class TErrorAttributes;
-
- static TEnricher Enricher_;
- static TFromExceptionEnricher FromExceptionEnricher_;
};
////////////////////////////////////////////////////////////////////////////////
diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h
index 87672ec6f6a..fa20fb06489 100644
--- a/library/cpp/yt/logging/logger-inl.h
+++ b/library/cpp/yt/logging/logger-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "logger.h"
#endif
-#undef LOGGER_INL_H_
#include <library/cpp/yt/yson_string/convert.h>
#include <library/cpp/yt/yson_string/string.h>
diff --git a/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h b/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h
index d91d20f2fd6..4211806880f 100644
--- a/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h
+++ b/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "atomic_intrusive_ptr.h"
#endif
-#undef ATOMIC_INTRUSIVE_PTR_INL_H_
#include <util/system/spinlock.h>
diff --git a/library/cpp/yt/system/cpu_id-inl.h b/library/cpp/yt/system/cpu_id-inl.h
index e336ba5a2ce..804995f5973 100644
--- a/library/cpp/yt/system/cpu_id-inl.h
+++ b/library/cpp/yt/system/cpu_id-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "cpu_id.h"
#endif
-#undef CPU_ID_INL_H_
#ifdef __linux__
#include <library/cpp/yt/rseq/rseq.h>
diff --git a/library/cpp/yt/threading/event_count-inl.h b/library/cpp/yt/threading/event_count-inl.h
index 18f7bfec52f..b366f93bb0f 100644
--- a/library/cpp/yt/threading/event_count-inl.h
+++ b/library/cpp/yt/threading/event_count-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "event_count.h"
#endif
-#undef EVENT_COUNT_INL_H_
#include <library/cpp/yt/assert/assert.h>
diff --git a/library/cpp/yt/threading/recursive_spin_lock-inl.h b/library/cpp/yt/threading/recursive_spin_lock-inl.h
index f692999f0b1..28b233efa94 100644
--- a/library/cpp/yt/threading/recursive_spin_lock-inl.h
+++ b/library/cpp/yt/threading/recursive_spin_lock-inl.h
@@ -3,7 +3,6 @@
// For the sake of sane code completion.
#include "recursive_spin_lock.h"
#endif
-#undef RECURSIVE_SPIN_LOCK_INL_H_
#include "spin_wait.h"
diff --git a/library/cpp/yt/threading/rw_spin_lock-inl.h b/library/cpp/yt/threading/rw_spin_lock-inl.h
index c6043e8f23a..167c95edfc5 100644
--- a/library/cpp/yt/threading/rw_spin_lock-inl.h
+++ b/library/cpp/yt/threading/rw_spin_lock-inl.h
@@ -4,7 +4,6 @@
// For the sake of sane code completion.
#include "rw_spin_lock.h"
#endif
-#undef RW_SPIN_LOCK_INL_H_
#include "spin_wait.h"
diff --git a/library/cpp/yt/threading/spin_lock-inl.h b/library/cpp/yt/threading/spin_lock-inl.h
index eedb1e52856..7e629ae3381 100644
--- a/library/cpp/yt/threading/spin_lock-inl.h
+++ b/library/cpp/yt/threading/spin_lock-inl.h
@@ -4,7 +4,6 @@
// For the sake of sane code completion.
#include "spin_lock.h"
#endif
-#undef SPIN_LOCK_INL_H_
#include "spin_wait.h"
diff --git a/library/cpp/yt/threading/spin_lock_base-inl.h b/library/cpp/yt/threading/spin_lock_base-inl.h
index 7135d081112..2e16a4e01f9 100644
--- a/library/cpp/yt/threading/spin_lock_base-inl.h
+++ b/library/cpp/yt/threading/spin_lock_base-inl.h
@@ -4,7 +4,6 @@
// For the sake of sane code completion.
#include "spin_lock_base.h"
#endif
-#undef SPIN_LOCK_BASE_INL_H_
#include <library/cpp/yt/misc/source_location.h>
diff --git a/library/cpp/yt/threading/spin_lock_count-inl.h b/library/cpp/yt/threading/spin_lock_count-inl.h
index 6ba1782052f..8100f2f3e73 100644
--- a/library/cpp/yt/threading/spin_lock_count-inl.h
+++ b/library/cpp/yt/threading/spin_lock_count-inl.h
@@ -2,7 +2,6 @@
#ifndef SPIN_LOCK_COUNT_INL_H_
#error "Direct inclusion of this file is not allowed, include spin_lock_count.h"
#endif
-#undef SPIN_LOCK_COUNT_INL_H_
#include <util/system/compiler.h>
diff --git a/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h b/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h
index b880135f244..590e3238a9d 100644
--- a/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h
+++ b/library/cpp/yt/threading/writer_starving_rw_spin_lock-inl.h
@@ -4,7 +4,6 @@
// For the sake of sane code completion.
#include "writer_starving_rw_spin_lock.h"
#endif
-#undef WRITER_STARVING_RW_SPIN_LOCK_INL_H_
#include "spin_wait.h"
diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp
index 696da0ebaec..f06b35f9bdb 100644
--- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp
+++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp
@@ -138,7 +138,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no
};
// adds _yql_sys_tablekeyswitch column which is required for outer lambda
- // _yql_sys_tableswitch equals "true" when reduce key is changed
+ // _yql_sys_tablekeyswitch equals "true" when reduce key is changed
TExprNode::TPtr keySwitchLambda = ctx.Builder(node.Pos())
.Lambda()
.Param("stream")
@@ -564,8 +564,6 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduceWithTrivialMa
.Reducer(newReduceLambda)
.Settings(newSettings)
.Done();
-
- return node;
}
TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseInnerMap(TExprBase node, TExprContext& ctx, const TGetParents& getParents) const {
diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp
index f81ee580f99..6c241b5c8dc 100644
--- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp
+++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.cpp
@@ -61,7 +61,7 @@ TYtSectionList ConvertInputTable(TExprBase input, TExprContext& ctx, const TConv
mergedSettings = NYql::RemoveSetting(*mergedSettings, EYtSettingType::Unordered, ctx);
makeUnordered = false;
}
- if (!opts.KeepDirecRead_) {
+ if (!opts.KeepDirectRead_) {
mergedSettings = NYql::RemoveSetting(*mergedSettings, EYtSettingType::DirectRead, ctx);
}
if (opts.Settings_) {
@@ -775,7 +775,7 @@ TExprBase GetWorld(TExprBase input, TMaybeNode<TExprBase> main, TExprContext& ct
}
TConvertInputOpts::TConvertInputOpts()
- : KeepDirecRead_(false)
+ : KeepDirectRead_(false)
, MakeUnordered_(false)
, ClearUnordered_(false)
{
@@ -817,8 +817,8 @@ TConvertInputOpts::ExplicitFields(const TStructExprType& type, TPositionHandle p
}
TConvertInputOpts&
-TConvertInputOpts::KeepDirecRead(bool keepDirecRead) {
- KeepDirecRead_ = keepDirecRead;
+TConvertInputOpts::KeepDirectRead(bool keepDirectRead) {
+ KeepDirectRead_ = keepDirectRead;
return *this;
}
diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h
index 04430f7ac23..291638037a1 100644
--- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h
+++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_helper.h
@@ -57,7 +57,7 @@ NNodes::TExprBase GetWorld(NNodes::TExprBase input, NNodes::TMaybeNode<NNodes::T
struct TConvertInputOpts {
NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> Settings_;
NNodes::TMaybeNode<NNodes::TCoAtomList> CustomFields_;
- bool KeepDirecRead_;
+ bool KeepDirectRead_;
bool MakeUnordered_;
bool ClearUnordered_;
@@ -71,7 +71,7 @@ struct TConvertInputOpts {
TConvertInputOpts& ExplicitFields(const TStructExprType& type, TPositionHandle pos, TExprContext& ctx);
- TConvertInputOpts& KeepDirecRead(bool keepDirecRead = true);
+ TConvertInputOpts& KeepDirectRead(bool keepDirectRead = true);
TConvertInputOpts& MakeUnordered(bool makeUnordered = true);
diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp
index 8bf71c90547..02d5b8daab0 100644
--- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp
+++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp
@@ -388,7 +388,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::TakeOrSkip(TExprBase no
.Input<TYtReadTable>()
.World(ApplySyncListToWorld(GetWorld(input, {}, ctx).Ptr(), syncList, ctx))
.DataSource(GetDataSource(input, ctx))
- .Input(ConvertInputTable(input, ctx, TConvertInputOpts().KeepDirecRead(true).Settings(settings)))
+ .Input(ConvertInputTable(input, ctx, TConvertInputOpts().KeepDirectRead(true).Settings(settings)))
.Build()
.Done();
return KeepColumnOrder(res.Ptr(), node.Ref(), ctx, *State_->Types);
@@ -1003,7 +1003,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::PushPruneKeysIntoYtOper
State_->Configuration->RuntimeClusterSelection.Get().GetOrElse(DEFAULT_RUNTIME_CLUSTER_SELECTION);
auto cluster = DeriveClusterFromInput(op.Input(), selectionMode);
if (!cluster || !IsYtCompleteIsolatedLambda(extractorLambda.Ref(), syncList, *cluster, false, selectionMode)) {
- return {};
+ return node;
}
auto outItemType = SilentGetSequenceItemType(op.Input().Ref(), true);
diff --git a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp
index dda15abc3db..67288ff28ed 100644
--- a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp
@@ -438,11 +438,11 @@ public:
const auto enableDynamicStoreRead = ytState->Configuration->EnableDynamicStoreReadInDQ.Get().GetOrElse(false);
ui64 chunksCount = 0ull;
for (auto section: maybeRead.Cast().Input()) {
- if (HasSettingsExcept(maybeRead.Cast().Input().Item(0).Settings().Ref(), DqReadSupportedSettings) || HasNonEmptyKeyFilter(maybeRead.Cast().Input().Item(0))) {
+ if (HasSettingsExcept(section.Settings().Ref(), DqReadSupportedSettings) || HasNonEmptyKeyFilter(section)) {
TStringBuilder info;
info << "unsupported path settings: ";
- if (maybeRead.Cast().Input().Item(0).Settings().Size() > 0) {
- for (auto& setting : maybeRead.Cast().Input().Item(0).Settings().Ref().Children()) {
+ if (section.Settings().Size() > 0) {
+ for (auto& setting : section.Settings().Ref().Children()) {
if (setting->ChildrenSize() != 0) {
info << setting->Child(0)->Content() << ",";
}
@@ -678,7 +678,7 @@ public:
++idx;
continue;
}
- ui64 readSize = std::accumulate(res[idx].begin(), res[idx].end(), 0);
+ ui64 readSize = std::accumulate(res[idx].begin(), res[idx].end(), 0ull);
++idx;
dataSizePerJob = Max(ui64(dataSizePerJob / *codecCpu), 10_KB);
const ui64 parts = (readSize + dataSizePerJob - 1) / dataSizePerJob;
diff --git a/yt/yql/providers/yt/provider/yql_yt_key.cpp b/yt/yql/providers/yt/provider/yql_yt_key.cpp
index 54b5eaf2b16..f1aa1bf18d9 100644
--- a/yt/yql/providers/yt/provider/yql_yt_key.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_key.cpp
@@ -187,6 +187,7 @@ bool TYtKey::Parse(const TExprNode& key, TExprContext& ctx, bool isOutput) {
} else {
ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), "Expected String or MrTableRange[Strict]"));
+ return false;
}
if (key.ChildrenSize() > 1) {
@@ -225,16 +226,6 @@ bool TYtKey::Parse(const TExprNode& key, TExprContext& ctx, bool isOutput) {
return false;
}
ExtraColumns = value;
- } else if (tag.IsAtom("typeId")) {
- if (Type != EType::View) {
- ctx.AddError(TIssue(ctx.GetPosition(tag.Pos()), "Missed object id."));
- return false;
- }
- if (const auto viewNode = key.Child(i)->Child(1);
- !(viewNode->IsCallable("String") && viewNode->ChildrenSize() == 1 && viewNode->Head().IsAtom("VIEW"))) {
- ctx.AddError(TIssue(ctx.GetPosition(viewNode->Pos()), "Expected String 'VIEW'."));
- return false;
- }
} else {
ctx.AddError(TIssue(ctx.GetPosition(tag.Pos()), TStringBuilder() << "Unexpected tag: " << tag.Content()));
return false;
diff --git a/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp b/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp
index a2afabd6b84..e701cd14979 100644
--- a/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_layers_integration.cpp
@@ -11,7 +11,12 @@ public:
TVector<std::pair<TKey, const TLayerInfo*>> infos;
infos.reserve(order.size());
for (auto &e: order) {
- infos.emplace_back(e, Data_.FindPtr(e));
+ auto info = Data_.FindPtr(e);
+ if (!info) {
+ ReportUnknownLayer(e, ctx);
+ return {};
+ }
+ infos.emplace_back(e, info);
if (infos.back().second->Locations.empty()) {
ctx.AddError(NYql::TIssue(
TStringBuilder() << "No locations for layer " << e.ToString() << ", consider adding it using yt.LayerCaches"
@@ -41,10 +46,7 @@ public:
bool UpdateLayerLocations(const TKey& key, TVector<TLocation>&& locs, NYql::TExprContext& ctx) override {
auto ptr = Data_.FindPtr(key);
if (!ptr) {
- ctx.AddError(NYql::TIssue(
- TStringBuilder() << "Layer with key=(Name=" << key.Name.GetOrElse("nullopt").Quote()
- << ", Url=" << key.Url.GetOrElse("nullopt").Quote() << ") not found"
- ));
+ ReportUnknownLayer(key, ctx);
return false;
}
ptr->Locations = std::move(locs);
@@ -56,6 +58,12 @@ public:
}
private:
+ static void ReportUnknownLayer(const TKey& key, NYql::TExprContext& ctx) {
+ ctx.AddError(NYql::TIssue(
+ TStringBuilder() << "Layer with key=(Name=" << key.Name.GetOrElse("nullopt").Quote()
+ << ", Url=" << key.Url.GetOrElse("nullopt").Quote() << ") not found"
+ ));
+ }
THashMap<TKey, TLayerInfo> Data_;
};
}
diff --git a/yt/yt/core/yson/producer-inl.h b/yt/yt/core/yson/producer-inl.h
index 7cfe5ff572c..034b972a234 100644
--- a/yt/yt/core/yson/producer-inl.h
+++ b/yt/yt/core/yson/producer-inl.h
@@ -11,8 +11,8 @@ namespace NYT::NYson {
////////////////////////////////////////////////////////////////////////////////
template <class... TAdditionalArgs>
-TExtendedYsonProducer<TAdditionalArgs...>::TExtendedYsonProducer(
- TExtendedYsonProducer::TUnderlyingCallback callback,
+TParametricYsonProducer<TAdditionalArgs...>::TParametricYsonProducer(
+ TParametricYsonProducer::TUnderlyingCallback callback,
EYsonType type)
: Type_(type)
, Callback_(std::move(callback))
@@ -21,11 +21,11 @@ TExtendedYsonProducer<TAdditionalArgs...>::TExtendedYsonProducer(
}
template <class... TAdditionalArgs>
-void TExtendedYsonProducer<TAdditionalArgs...>::Run(IYsonConsumer* consumer, TAdditionalArgs... args) const
+void TParametricYsonProducer<TAdditionalArgs...>::Run(IYsonConsumer* consumer, TAdditionalArgs... args) const
{
Callback_(consumer, std::forward<TAdditionalArgs>(args)...);
}
////////////////////////////////////////////////////////////////////////////////
-} // namespace NYT::NYson \ No newline at end of file
+} // namespace NYT::NYson
diff --git a/yt/yt/core/yson/producer.cpp b/yt/yt/core/yson/producer.cpp
index aeff4b7266a..4951612d786 100644
--- a/yt/yt/core/yson/producer.cpp
+++ b/yt/yt/core/yson/producer.cpp
@@ -4,20 +4,6 @@ namespace NYT::NYson {
////////////////////////////////////////////////////////////////////////////////
-TYsonProducer::TYsonProducer(TYsonCallback callback, EYsonType type)
- : Type_(type)
- , Callback_(std::move(callback))
-{
- YT_ASSERT(Callback_);
-}
-
-void TYsonProducer::Run(IYsonConsumer* consumer) const
-{
- Callback_(consumer);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
void Serialize(const TYsonProducer& value, IYsonConsumer* consumer)
{
value.Run(consumer);
diff --git a/yt/yt/core/yson/producer.h b/yt/yt/core/yson/producer.h
index 7310d1540ba..5ddae14f540 100644
--- a/yt/yt/core/yson/producer.h
+++ b/yt/yt/core/yson/producer.h
@@ -12,45 +12,26 @@ namespace NYT::NYson {
////////////////////////////////////////////////////////////////////////////////
//! A callback capable of generating YSON by calling appropriate
-//! methods for its IYsonConsumer argument.
-using TYsonCallback = TCallback<void(IYsonConsumer*)>;
-
-//! A callback capable of generating YSON by calling appropriate
//! methods for its IYsonConsumer and some additional arguments.
template <class... TAdditionalArgs>
-using TExtendedYsonCallback = TCallback<void(IYsonConsumer*, TAdditionalArgs...)>;
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TYsonProducer
-{
-public:
- DEFINE_BYVAL_RO_PROPERTY(NYson::EYsonType, Type);
+using TParametricYsonCallback = TCallback<void(IYsonConsumer*, TAdditionalArgs...)>;
-public:
- TYsonProducer() = default;
- TYsonProducer(
- TYsonCallback callback,
- EYsonType type = NYson::EYsonType::Node);
-
- void Run(IYsonConsumer* consumer) const;
-
-private:
- TYsonCallback Callback_;
-};
+//! A callback capable of generating YSON by calling appropriate
+//! methods for its IYsonConsumer argument.
+using TYsonCallback = TParametricYsonCallback<>;
////////////////////////////////////////////////////////////////////////////////
template <class... TAdditionalArgs>
-class TExtendedYsonProducer
+class TParametricYsonProducer
{
- using TUnderlyingCallback = TExtendedYsonCallback<TAdditionalArgs...>;
+ using TUnderlyingCallback = TParametricYsonCallback<TAdditionalArgs...>;
public:
DEFINE_BYVAL_RO_PROPERTY(NYson::EYsonType, Type);
public:
- TExtendedYsonProducer() = default;
- TExtendedYsonProducer(
+ TParametricYsonProducer() = default;
+ TParametricYsonProducer(
TUnderlyingCallback callback,
EYsonType type = NYson::EYsonType::Node);
@@ -60,6 +41,9 @@ private:
TUnderlyingCallback Callback_;
};
+//! A producer generating a plain YSON with no additional arguments.
+using TYsonProducer = TParametricYsonProducer<>;
+
////////////////////////////////////////////////////////////////////////////////
void Serialize(const TYsonProducer& value, NYson::IYsonConsumer* consumer);
diff --git a/yt/yt/core/yson/public.h b/yt/yt/core/yson/public.h
index b322e9007f1..70bd806d5a7 100644
--- a/yt/yt/core/yson/public.h
+++ b/yt/yt/core/yson/public.h
@@ -12,9 +12,9 @@ namespace NYT::NYson {
enum class ETokenType;
-class TYsonProducer;
template <class... TAdditionalArgs>
-class TExtendedYsonProducer;
+class TParametricYsonProducer;
+using TYsonProducer = TParametricYsonProducer<>;
class TYsonInput;
class TYsonOutput;
diff --git a/yt/yt/core/ytree/ypath_service.cpp b/yt/yt/core/ytree/ypath_service.cpp
index 3d3e4185617..16460f9b0cb 100644
--- a/yt/yt/core/ytree/ypath_service.cpp
+++ b/yt/yt/core/ytree/ypath_service.cpp
@@ -233,7 +233,7 @@ class TFromExtendedProducerYPathService
: public TYPathServiceBase
, public TSupportsGet
{
- using TUnderlyingProducer = TExtendedYsonProducer<const IAttributeDictionaryPtr&>;
+ using TUnderlyingProducer = TParametricYsonProducer<const IAttributeDictionaryPtr&>;
public:
explicit TFromExtendedProducerYPathService(TUnderlyingProducer producer)
: Producer_(std::move(producer))
@@ -326,7 +326,7 @@ private:
};
IYPathServicePtr IYPathService::FromProducer(
- NYson::TExtendedYsonProducer<const IAttributeDictionaryPtr&> producer)
+ NYson::TParametricYsonProducer<const IAttributeDictionaryPtr&> producer)
{
return New<TFromExtendedProducerYPathService>(std::move(producer));
}
diff --git a/yt/yt/core/ytree/ypath_service.h b/yt/yt/core/ytree/ypath_service.h
index 6df44dba804..dbd655c2688 100644
--- a/yt/yt/core/ytree/ypath_service.h
+++ b/yt/yt/core/ytree/ypath_service.h
@@ -106,7 +106,7 @@ struct IYPathService
* an ephemeral tree, and the request is forwarded to that tree.
*/
static IYPathServicePtr FromProducer(
- NYson::TExtendedYsonProducer<const IAttributeDictionaryPtr&> producer);
+ NYson::TParametricYsonProducer<const IAttributeDictionaryPtr&> producer);
//! Creates a producer from YPath service.
/*!