diff options
| author | robot-piglet <[email protected]> | 2025-11-28 14:23:46 +0300 |
|---|---|---|
| committer | robot-piglet <[email protected]> | 2025-11-28 15:10:40 +0300 |
| commit | ba6a7a7520ce0c3d80e7630aa85ff3710174b77b (patch) | |
| tree | e86753db161ec749a6742fe3b22227d3d966097b /contrib/python | |
| parent | 3b2b3764085395eaf6049ae0902869c5e7861fd5 (diff) | |
Intermediate changes
commit_hash:e2703b06031e348778f8950a45ef4d94aaefe57b
Diffstat (limited to 'contrib/python')
24 files changed, 543 insertions, 32 deletions
diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA index 3df1dc323cb..1ef7c68f9ff 100644 --- a/contrib/python/ydb/py3/.dist-info/METADATA +++ b/contrib/python/ydb/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ydb -Version: 3.21.13 +Version: 3.22.0 Summary: YDB Python SDK Home-page: http://github.com/ydb-platform/ydb-python-sdk Author: Yandex LLC diff --git a/contrib/python/ydb/py3/patches/05-change-import-protobufs-query.patch b/contrib/python/ydb/py3/patches/05-change-import-protobufs-query.patch index 1db5f29ac66..9386f4b3ef7 100644 --- a/contrib/python/ydb/py3/patches/05-change-import-protobufs-query.patch +++ b/contrib/python/ydb/py3/patches/05-change-import-protobufs-query.patch @@ -1,9 +1,9 @@ --- contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query.py (index) +++ contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query.py (working tree) -@@ -3,11 +3,10 @@ import typing +@@ -2,11 +2,10 @@ from dataclasses import dataclass + import typing from typing import Optional - -# Workaround for good IDE and universal for runtime -if typing.TYPE_CHECKING: - from ..v4.protos import ydb_query_pb2 @@ -15,22 +15,22 @@ + from contrib.ydb.public.api.protos import ydb_query_pb2 from . import ydb_query_public_types as public_types - + --- contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query_public_types.py (index) +++ contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query_public_types.py (working tree) -@@ -3,11 +3,10 @@ import typing +@@ -4,11 +4,10 @@ import typing - from .common_utils import IToProto + from .common_utils import IFromProto, IToProto -# Workaround for good IDE and universal for runtime -if typing.TYPE_CHECKING: -- from ..v4.protos import ydb_query_pb2 +- from ..v4.protos import ydb_query_pb2, ydb_formats_pb2 -else: -- from ..common.protos import ydb_query_pb2 +- from ..common.protos import ydb_query_pb2, ydb_formats_pb2 +try: -+ from ydb.public.api.protos import ydb_query_pb2 ++ from ydb.public.api.protos import ydb_query_pb2, ydb_formats_pb2 +except ImportError: -+ from contrib.ydb.public.api.protos import ydb_query_pb2 ++ from contrib.ydb.public.api.protos import ydb_query_pb2, ydb_formats_pb2 class BaseQueryTxMode(IToProto): diff --git a/contrib/python/ydb/py3/patches/06-change-import-protobufs-coordination.patch b/contrib/python/ydb/py3/patches/06-change-import-protobufs-coordination.patch new file mode 100644 index 00000000000..72e51f2e10a --- /dev/null +++ b/contrib/python/ydb/py3/patches/06-change-import-protobufs-coordination.patch @@ -0,0 +1,34 @@ +--- contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination.py (index) ++++ contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination.py (working tree) +@@ -3,10 +3,10 @@ from dataclasses import dataclass + + from .ydb_coordination_public_types import NodeConfig + +-if typing.TYPE_CHECKING: +- from ..v4.protos import ydb_coordination_pb2 +-else: +- from ..common.protos import ydb_coordination_pb2 ++try: ++ from ydb.public.api.protos import ydb_coordination_pb2 ++except ImportError: ++ from contrib.ydb.public.api.protos import ydb_coordination_pb2 + + from .common_utils import IToProto + +--- contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination_public_types.py (index) ++++ contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination_public_types.py (working tree) +@@ -3,10 +3,10 @@ from enum import IntEnum + import typing + + +-if typing.TYPE_CHECKING: +- from ..v4.protos import ydb_coordination_pb2 +-else: +- from ..common.protos import ydb_coordination_pb2 ++try: ++ from ydb.public.api.protos import ydb_coordination_pb2 ++except ImportError: ++ from contrib.ydb.public.api.protos import ydb_coordination_pb2 + + + class ConsistencyMode(IntEnum):
\ No newline at end of file diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make index 892ad0c26d8..c7ec357b7e9 100644 --- a/contrib/python/ydb/py3/ya.make +++ b/contrib/python/ydb/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.21.13) +VERSION(3.22.0) LICENSE(Apache-2.0) @@ -30,6 +30,8 @@ PY_SRCS( ydb/_grpc/common/__init__.py ydb/_grpc/grpcwrapper/__init__.py ydb/_grpc/grpcwrapper/common_utils.py + ydb/_grpc/grpcwrapper/ydb_coordination.py + ydb/_grpc/grpcwrapper/ydb_coordination_public_types.py ydb/_grpc/grpcwrapper/ydb_query.py ydb/_grpc/grpcwrapper/ydb_query_public_types.py ydb/_grpc/grpcwrapper/ydb_scheme.py @@ -54,6 +56,8 @@ PY_SRCS( ydb/aio/__init__.py ydb/aio/_utilities.py ydb/aio/connection.py + ydb/aio/coordination/__init__.py + ydb/aio/coordination/client.py ydb/aio/credentials.py ydb/aio/driver.py ydb/aio/iam.py @@ -70,6 +74,9 @@ PY_SRCS( ydb/auth_helpers.py ydb/connection.py ydb/convert.py + ydb/coordination/__init__.py + ydb/coordination/base.py + ydb/coordination/client.py ydb/credentials.py ydb/dbapi/__init__.py ydb/dbapi/connection.py diff --git a/contrib/python/ydb/py3/ydb/_apis.py b/contrib/python/ydb/py3/ydb/_apis.py index b0fa8f3cb22..96e8b74940a 100644 --- a/contrib/python/ydb/py3/ydb/_apis.py +++ b/contrib/python/ydb/py3/ydb/_apis.py @@ -10,6 +10,7 @@ try: ydb_operation_v1_pb2_grpc, ydb_topic_v1_pb2_grpc, ydb_query_v1_pb2_grpc, + ydb_coordination_v1_pb2_grpc, ) from ydb.public.api.protos import ( @@ -21,6 +22,7 @@ try: ydb_operation_pb2, ydb_common_pb2, ydb_query_pb2, + ydb_coordination_pb2, ) except ImportError: @@ -32,6 +34,7 @@ except ImportError: ydb_operation_v1_pb2_grpc, ydb_topic_v1_pb2_grpc, ydb_query_v1_pb2_grpc, + ydb_coordination_v1_pb2_grpc, ) from contrib.ydb.public.api.protos import ( @@ -43,6 +46,7 @@ except ImportError: ydb_operation_pb2, ydb_common_pb2, ydb_query_pb2, + ydb_coordination_pb2, ) @@ -55,6 +59,7 @@ ydb_table = ydb_table_pb2 ydb_discovery = ydb_discovery_pb2 ydb_operation = ydb_operation_pb2 ydb_query = ydb_query_pb2 +ydb_coordination = ydb_coordination_pb2 class CmsService(object): @@ -133,3 +138,13 @@ class QueryService(object): ExecuteQuery = "ExecuteQuery" ExecuteScript = "ExecuteScript" FetchScriptResults = "FetchScriptResults" + + +class CoordinationService(object): + Stub = ydb_coordination_v1_pb2_grpc.CoordinationServiceStub + + Session = "Session" + CreateNode = "CreateNode" + AlterNode = "AlterNode" + DropNode = "DropNode" + DescribeNode = "DescribeNode" diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination.py new file mode 100644 index 00000000000..fa48c32b37b --- /dev/null +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination.py @@ -0,0 +1,57 @@ +import typing +from dataclasses import dataclass + +from .ydb_coordination_public_types import NodeConfig + +try: + from ydb.public.api.protos import ydb_coordination_pb2 +except ImportError: + from contrib.ydb.public.api.protos import ydb_coordination_pb2 + +from .common_utils import IToProto + + +@dataclass +class CreateNodeRequest(IToProto): + path: str + config: typing.Optional[NodeConfig] + + def to_proto(self) -> ydb_coordination_pb2.CreateNodeRequest: + cfg_proto = self.config.to_proto() if self.config else None + return ydb_coordination_pb2.CreateNodeRequest( + path=self.path, + config=cfg_proto, + ) + + +@dataclass +class AlterNodeRequest(IToProto): + path: str + config: NodeConfig + + def to_proto(self) -> ydb_coordination_pb2.AlterNodeRequest: + cfg_proto = self.config.to_proto() if self.config else None + return ydb_coordination_pb2.AlterNodeRequest( + path=self.path, + config=cfg_proto, + ) + + +@dataclass +class DescribeNodeRequest(IToProto): + path: str + + def to_proto(self) -> ydb_coordination_pb2.DescribeNodeRequest: + return ydb_coordination_pb2.DescribeNodeRequest( + path=self.path, + ) + + +@dataclass +class DropNodeRequest(IToProto): + path: str + + def to_proto(self) -> ydb_coordination_pb2.DropNodeRequest: + return ydb_coordination_pb2.DropNodeRequest( + path=self.path, + ) diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination_public_types.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination_public_types.py new file mode 100644 index 00000000000..1199a6a98a1 --- /dev/null +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_coordination_public_types.py @@ -0,0 +1,57 @@ +from dataclasses import dataclass +from enum import IntEnum +import typing + + +try: + from ydb.public.api.protos import ydb_coordination_pb2 +except ImportError: + from contrib.ydb.public.api.protos import ydb_coordination_pb2 + + +class ConsistencyMode(IntEnum): + UNSET = ydb_coordination_pb2.ConsistencyMode.CONSISTENCY_MODE_UNSET + STRICT = ydb_coordination_pb2.ConsistencyMode.CONSISTENCY_MODE_STRICT + RELAXED = ydb_coordination_pb2.ConsistencyMode.CONSISTENCY_MODE_RELAXED + + +class RateLimiterCountersMode(IntEnum): + UNSET = ydb_coordination_pb2.RateLimiterCountersMode.RATE_LIMITER_COUNTERS_MODE_UNSET + AGGREGATED = ydb_coordination_pb2.RateLimiterCountersMode.RATE_LIMITER_COUNTERS_MODE_AGGREGATED + DETAILED = ydb_coordination_pb2.RateLimiterCountersMode.RATE_LIMITER_COUNTERS_MODE_DETAILED + + +@dataclass +class NodeConfig: + attach_consistency_mode: ConsistencyMode + rate_limiter_counters_mode: RateLimiterCountersMode + read_consistency_mode: ConsistencyMode + self_check_period_millis: int + session_grace_period_millis: int + + @staticmethod + def from_proto(msg: ydb_coordination_pb2.Config) -> "NodeConfig": + return NodeConfig( + attach_consistency_mode=msg.attach_consistency_mode, + rate_limiter_counters_mode=msg.rate_limiter_counters_mode, + read_consistency_mode=msg.read_consistency_mode, + self_check_period_millis=msg.self_check_period_millis, + session_grace_period_millis=msg.session_grace_period_millis, + ) + + def to_proto(self) -> ydb_coordination_pb2.Config: + return ydb_coordination_pb2.Config( + attach_consistency_mode=self.attach_consistency_mode, + rate_limiter_counters_mode=self.rate_limiter_counters_mode, + read_consistency_mode=self.read_consistency_mode, + self_check_period_millis=self.self_check_period_millis, + session_grace_period_millis=self.session_grace_period_millis, + ) + + +class DescribeResult: + @staticmethod + def from_proto(msg: ydb_coordination_pb2.DescribeNodeResponse) -> "NodeConfig": + result = ydb_coordination_pb2.DescribeNodeResult() + msg.operation.result.Unpack(result) + return NodeConfig.from_proto(result.config) diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query.py index 8fc09b00723..6056f9c1d10 100644 --- a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query.py +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query.py @@ -2,7 +2,6 @@ from dataclasses import dataclass import typing from typing import Optional - try: from ydb.public.api.protos import ydb_query_pb2 except ImportError: @@ -167,15 +166,24 @@ class ExecuteQueryRequest(IToProto): exec_mode: int parameters: dict stats_mode: int + schema_inclusion_mode: int + result_set_format: int + arrow_format_settings: Optional[public_types.ArrowFormatSettings] def to_proto(self) -> ydb_query_pb2.ExecuteQueryRequest: tx_control = self.tx_control.to_proto() if self.tx_control is not None else self.tx_control + arrow_format_settings = ( + self.arrow_format_settings.to_proto() if self.arrow_format_settings is not None else None + ) return ydb_query_pb2.ExecuteQueryRequest( session_id=self.session_id, tx_control=tx_control, query_content=self.query_content.to_proto(), exec_mode=self.exec_mode, stats_mode=self.stats_mode, + schema_inclusion_mode=self.schema_inclusion_mode, + result_set_format=self.result_set_format, + arrow_format_settings=arrow_format_settings, concurrent_result_sets=self.concurrent_result_sets, parameters=convert.query_parameters_to_pb(self.parameters), ) diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query_public_types.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query_public_types.py index 24f3db30bc4..02697cab939 100644 --- a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query_public_types.py +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_query_public_types.py @@ -1,12 +1,13 @@ import abc +import enum import typing -from .common_utils import IToProto +from .common_utils import IFromProto, IToProto try: - from ydb.public.api.protos import ydb_query_pb2 + from ydb.public.api.protos import ydb_query_pb2, ydb_formats_pb2 except ImportError: - from contrib.ydb.public.api.protos import ydb_query_pb2 + from contrib.ydb.public.api.protos import ydb_query_pb2, ydb_formats_pb2 class BaseQueryTxMode(IToProto): @@ -92,3 +93,51 @@ class QueryStaleReadOnly(BaseQueryTxMode): def to_proto(self) -> ydb_query_pb2.StaleModeSettings: return ydb_query_pb2.StaleModeSettings() + + +class ArrowCompressionCodecType(enum.IntEnum): + UNSPECIFIED = 0 + NONE = 1 + ZSTD = 2 + LZ4_FRAME = 3 + + +class ArrowCompressionCodec(IToProto): + """Compression codec for Arrow format result sets.""" + + def __init__( + self, codec_type: typing.Optional[ArrowCompressionCodecType] = None, level: typing.Optional[int] = None + ): + self.type = codec_type if codec_type is not None else ArrowCompressionCodecType.UNSPECIFIED + self.level = level + + def to_proto(self): + return ydb_formats_pb2.ArrowFormatSettings.CompressionCodec(type=self.type, level=self.level) + + +class ArrowFormatSettings(IToProto): + """Settings for Arrow format result sets.""" + + def __init__(self, compression_codec: typing.Optional[ArrowCompressionCodec] = None): + self.compression_codec = compression_codec + + def to_proto(self): + settings = ydb_formats_pb2.ArrowFormatSettings() + if self.compression_codec is not None: + codec_proto = self.compression_codec.to_proto() + settings.compression_codec.CopyFrom(codec_proto) + return settings + + +class ArrowFormatMeta(IFromProto): + """Metadata for Arrow format result sets containing the schema.""" + + def __init__(self, schema: bytes): + self.schema = schema + + @classmethod + def from_proto(cls, proto_message): + return cls(schema=proto_message.schema) + + def __repr__(self): + return f"ArrowFormatMeta(schema_size={len(self.schema)} bytes)" diff --git a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py index aae3a0a1239..71d9850a373 100644 --- a/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py +++ b/contrib/python/ydb/py3/ydb/_grpc/grpcwrapper/ydb_topic.py @@ -478,18 +478,12 @@ class StreamReadMessage: def to_proto( self, ) -> ydb_topic_pb2.StreamReadMessage.InitRequest.TopicReadSettings: - res = ydb_topic_pb2.StreamReadMessage.InitRequest.TopicReadSettings() - res.path = self.path - res.partition_ids.extend(self.partition_ids) - max_lag = proto_duration_from_timedelta(self.max_lag) - if max_lag is not None: - res.max_lag = max_lag - - read_from = proto_timestamp_from_datetime(self.read_from) - if read_from is not None: - res.read_from = read_from - - return res + return ydb_topic_pb2.StreamReadMessage.InitRequest.TopicReadSettings( + path=self.path, + partition_ids=self.partition_ids, + max_lag=proto_duration_from_timedelta(self.max_lag), + read_from=proto_timestamp_from_datetime(self.read_from), + ) @dataclass class InitResponse(IFromProto): diff --git a/contrib/python/ydb/py3/ydb/aio/__init__.py b/contrib/python/ydb/py3/ydb/aio/__init__.py index 1c9c887c229..4e4192a888e 100644 --- a/contrib/python/ydb/py3/ydb/aio/__init__.py +++ b/contrib/python/ydb/py3/ydb/aio/__init__.py @@ -1,3 +1,5 @@ from .driver import Driver # noqa from .table import SessionPool, retry_operation # noqa from .query import QuerySessionPool, QuerySession, QueryTxContext # noqa + +# from .coordination import CoordinationClient # noqa diff --git a/contrib/python/ydb/py3/ydb/aio/coordination/__init__.py b/contrib/python/ydb/py3/ydb/aio/coordination/__init__.py new file mode 100644 index 00000000000..f6d48237f98 --- /dev/null +++ b/contrib/python/ydb/py3/ydb/aio/coordination/__init__.py @@ -0,0 +1,5 @@ +__all__ = [ + "CoordinationClient", +] + +from .client import CoordinationClient diff --git a/contrib/python/ydb/py3/ydb/aio/coordination/client.py b/contrib/python/ydb/py3/ydb/aio/coordination/client.py new file mode 100644 index 00000000000..b36b8950b65 --- /dev/null +++ b/contrib/python/ydb/py3/ydb/aio/coordination/client.py @@ -0,0 +1,39 @@ +from typing import Optional + +from ..._grpc.grpcwrapper.ydb_coordination import ( + CreateNodeRequest, + DescribeNodeRequest, + AlterNodeRequest, + DropNodeRequest, +) +from ..._grpc.grpcwrapper.ydb_coordination_public_types import NodeConfig +from ...coordination.base import BaseCoordinationClient + + +class CoordinationClient(BaseCoordinationClient): + async def create_node(self, path: str, config: Optional[NodeConfig] = None, settings=None): + return await self._call_create( + CreateNodeRequest(path=path, config=config).to_proto(), + settings=settings, + ) + + async def describe_node(self, path: str, settings=None) -> NodeConfig: + return await self._call_describe( + DescribeNodeRequest(path=path).to_proto(), + settings=settings, + ) + + async def alter_node(self, path: str, new_config: NodeConfig, settings=None): + return await self._call_alter( + AlterNodeRequest(path=path, config=new_config).to_proto(), + settings=settings, + ) + + async def delete_node(self, path: str, settings=None): + return await self._call_delete( + DropNodeRequest(path=path).to_proto(), + settings=settings, + ) + + async def lock(self): + raise NotImplementedError("Will be implemented in future release") diff --git a/contrib/python/ydb/py3/ydb/aio/query/session.py b/contrib/python/ydb/py3/ydb/aio/query/session.py index 98ea1849561..01e6bb27c8a 100644 --- a/contrib/python/ydb/py3/ydb/aio/query/session.py +++ b/contrib/python/ydb/py3/ydb/aio/query/session.py @@ -124,6 +124,9 @@ class QuerySession(BaseQuerySession): settings: Optional[BaseRequestSettings] = None, *, stats_mode: Optional[base.QueryStatsMode] = None, + schema_inclusion_mode: Optional[base.QuerySchemaInclusionMode] = None, + result_set_format: Optional[base.QueryResultSetFormat] = None, + arrow_format_settings: Optional[base.ArrowFormatSettings] = None, ) -> AsyncResponseContextIterator: """Sends a query to Query Service @@ -138,6 +141,13 @@ class QuerySession(BaseQuerySession): 2) QueryStatsMode.BASIC; 3) QueryStatsMode.FULL; 4) QueryStatsMode.PROFILE; + :param schema_inclusion_mode: Schema inclusion mode for result sets: + 1) QuerySchemaInclusionMode.ALWAYS, which is default; + 2) QuerySchemaInclusionMode.FIRST_ONLY. + :param result_set_format: Format of the result sets: + 1) QueryResultSetFormat.VALUE, which is default; + 2) QueryResultSetFormat.ARROW. + :param arrow_format_settings: Settings for Arrow format when result_set_format is ARROW. :return: Iterator with result sets """ @@ -150,6 +160,9 @@ class QuerySession(BaseQuerySession): syntax=syntax, exec_mode=exec_mode, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, concurrent_result_sets=concurrent_result_sets, settings=settings, ) diff --git a/contrib/python/ydb/py3/ydb/aio/query/transaction.py b/contrib/python/ydb/py3/ydb/aio/query/transaction.py index 9b2db2efb46..d8e99ef313c 100644 --- a/contrib/python/ydb/py3/ydb/aio/query/transaction.py +++ b/contrib/python/ydb/py3/ydb/aio/query/transaction.py @@ -144,6 +144,9 @@ class QueryTxContext(BaseQueryTxContext): settings: Optional[BaseRequestSettings] = None, *, stats_mode: Optional[base.QueryStatsMode] = None, + schema_inclusion_mode: Optional[base.QuerySchemaInclusionMode] = None, + result_set_format: Optional[base.QueryResultSetFormat] = None, + arrow_format_settings: Optional[base.ArrowFormatSettings] = None, ) -> AsyncResponseContextIterator: """Sends a query to Query Service @@ -164,6 +167,13 @@ class QueryTxContext(BaseQueryTxContext): 2) QueryStatsMode.BASIC; 3) QueryStatsMode.FULL; 4) QueryStatsMode.PROFILE; + :param schema_inclusion_mode: Schema inclusion mode for result sets: + 1) QuerySchemaInclusionMode.ALWAYS, which is default; + 2) QuerySchemaInclusionMode.FIRST_ONLY. + :param result_set_format: Format of the result sets: + 1) QueryResultSetFormat.VALUE, which is default; + 2) QueryResultSetFormat.ARROW. + :param arrow_format_settings: Settings for Arrow format when result_set_format is ARROW. :return: Iterator with result sets """ @@ -176,6 +186,9 @@ class QueryTxContext(BaseQueryTxContext): syntax=syntax, exec_mode=exec_mode, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, concurrent_result_sets=concurrent_result_sets, settings=settings, ) diff --git a/contrib/python/ydb/py3/ydb/convert.py b/contrib/python/ydb/py3/ydb/convert.py index 485b58978b0..b54e8bddc8d 100644 --- a/contrib/python/ydb/py3/ydb/convert.py +++ b/contrib/python/ydb/py3/ydb/convert.py @@ -352,14 +352,19 @@ def _unwrap_optionality(column): class _ResultSet(object): - __slots__ = ("columns", "rows", "truncated", "snapshot", "index") + __slots__ = ("columns", "rows", "truncated", "snapshot", "index", "format", "arrow_format_meta", "data") - def __init__(self, columns, rows, truncated, snapshot=None, index=None): + def __init__( + self, columns, rows, truncated, snapshot=None, index=None, format=None, arrow_format_meta=None, data=None + ): self.columns = columns self.rows = rows self.truncated = truncated self.snapshot = snapshot self.index = index + self.format = format + self.arrow_format_meta = arrow_format_meta + self.data = data @classmethod def from_message(cls, message, table_client_settings=None, snapshot=None, index=None): @@ -385,12 +390,32 @@ class _ResultSet(object): column_parser, unwrapped_type = column_info row[column.name] = column_parser(unwrapped_type, value, table_client_settings) rows.append(row) - return cls(message.columns, rows, message.truncated, snapshot, index) + + from ydb.query import QueryResultSetFormat, ArrowFormatMeta + + result_format = message.format if message.format else QueryResultSetFormat.VALUE + + arrow_meta = None + if message.HasField("arrow_format_meta"): + arrow_meta = ArrowFormatMeta.from_proto(message.arrow_format_meta) + + data = message.data if message.data else None + + return cls(message.columns, rows, message.truncated, snapshot, index, result_format, arrow_meta, data) @classmethod def lazy_from_message(cls, message, table_client_settings=None, snapshot=None): + from ydb.query import QueryResultSetFormat, ArrowFormatMeta + rows = _LazyRows(message.rows, table_client_settings, message.columns) - return cls(message.columns, rows, message.truncated, snapshot) + result_format = message.format if message.format else QueryResultSetFormat.VALUE + + arrow_meta = None + if message.HasField("arrow_format_meta"): + arrow_meta = ArrowFormatMeta.from_proto(message.arrow_format_meta) + + data = message.data if message.data else None + return cls(message.columns, rows, message.truncated, snapshot, None, result_format, arrow_meta, data) ResultSet = _ResultSet diff --git a/contrib/python/ydb/py3/ydb/coordination/__init__.py b/contrib/python/ydb/py3/ydb/coordination/__init__.py new file mode 100644 index 00000000000..fd994c567b0 --- /dev/null +++ b/contrib/python/ydb/py3/ydb/coordination/__init__.py @@ -0,0 +1,16 @@ +__all__ = [ + "CoordinationClient", + "NodeConfig", + "ConsistencyMode", + "RateLimiterCountersMode", + "DescribeResult", +] + +from .client import CoordinationClient + +from .._grpc.grpcwrapper.ydb_coordination_public_types import ( + NodeConfig, + ConsistencyMode, + RateLimiterCountersMode, + DescribeResult, +) diff --git a/contrib/python/ydb/py3/ydb/coordination/base.py b/contrib/python/ydb/py3/ydb/coordination/base.py new file mode 100644 index 00000000000..0be7cb8f96a --- /dev/null +++ b/contrib/python/ydb/py3/ydb/coordination/base.py @@ -0,0 +1,65 @@ +from .. import _apis, issues +from .._grpc.grpcwrapper.ydb_coordination_public_types import NodeConfig, DescribeResult +import logging + + +logger = logging.getLogger(__name__) + + +def wrapper_create_node(rpc_state, response_pb): + issues._process_response(response_pb.operation) + + +def wrapper_describe_node(rpc_state, response_pb) -> NodeConfig: + issues._process_response(response_pb.operation) + return DescribeResult.from_proto(response_pb) + + +def wrapper_delete_node(rpc_state, response_pb): + issues._process_response(response_pb.operation) + + +def wrapper_alter_node(rpc_state, response_pb): + issues._process_response(response_pb.operation) + + +class BaseCoordinationClient: + def __init__(self, driver): + logger.warning("Experimental API: interface may change in future releases.") + self._driver = driver + + def _call_create(self, request, settings=None): + return self._driver( + request, + _apis.CoordinationService.Stub, + _apis.CoordinationService.CreateNode, + wrap_result=wrapper_create_node, + settings=settings, + ) + + def _call_describe(self, request, settings=None): + return self._driver( + request, + _apis.CoordinationService.Stub, + _apis.CoordinationService.DescribeNode, + wrap_result=wrapper_describe_node, + settings=settings, + ) + + def _call_alter(self, request, settings=None): + return self._driver( + request, + _apis.CoordinationService.Stub, + _apis.CoordinationService.AlterNode, + wrap_result=wrapper_alter_node, + settings=settings, + ) + + def _call_delete(self, request, settings=None): + return self._driver( + request, + _apis.CoordinationService.Stub, + _apis.CoordinationService.DropNode, + wrap_result=wrapper_delete_node, + settings=settings, + ) diff --git a/contrib/python/ydb/py3/ydb/coordination/client.py b/contrib/python/ydb/py3/ydb/coordination/client.py new file mode 100644 index 00000000000..549528d9d64 --- /dev/null +++ b/contrib/python/ydb/py3/ydb/coordination/client.py @@ -0,0 +1,39 @@ +from typing import Optional + +from .._grpc.grpcwrapper.ydb_coordination import ( + CreateNodeRequest, + DescribeNodeRequest, + AlterNodeRequest, + DropNodeRequest, +) +from .._grpc.grpcwrapper.ydb_coordination_public_types import NodeConfig +from .base import BaseCoordinationClient + + +class CoordinationClient(BaseCoordinationClient): + def create_node(self, path: str, config: Optional[NodeConfig], settings=None): + return self._call_create( + CreateNodeRequest(path=path, config=config).to_proto(), + settings=settings, + ) + + def describe_node(self, path: str, settings=None) -> NodeConfig: + return self._call_describe( + DescribeNodeRequest(path=path).to_proto(), + settings=settings, + ) + + def alter_node(self, path: str, new_config: NodeConfig, settings=None): + return self._call_alter( + AlterNodeRequest(path=path, config=new_config).to_proto(), + settings=settings, + ) + + def delete_node(self, path: str, settings=None): + return self._call_delete( + DropNodeRequest(path=path).to_proto(), + settings=settings, + ) + + def lock(self): + raise NotImplementedError("Will be implemented in future release") diff --git a/contrib/python/ydb/py3/ydb/query/__init__.py b/contrib/python/ydb/py3/ydb/query/__init__.py index 56e87524494..2814e50453d 100644 --- a/contrib/python/ydb/py3/ydb/query/__init__.py +++ b/contrib/python/ydb/py3/ydb/query/__init__.py @@ -10,6 +10,12 @@ __all__ = [ "QuerySession", "QueryStatsMode", "QueryTxContext", + "QuerySchemaInclusionMode", + "QueryResultSetFormat", + "ArrowCompressionCodecType", + "ArrowCompressionCodec", + "ArrowFormatSettings", + "ArrowFormatMeta", ] import logging @@ -18,6 +24,8 @@ from .base import ( QueryClientSettings, QueryExplainResultFormat, QueryStatsMode, + QuerySchemaInclusionMode, + QueryResultSetFormat, ) from .session import QuerySession @@ -30,6 +38,10 @@ from .._grpc.grpcwrapper.ydb_query_public_types import ( QuerySerializableReadWrite, QuerySnapshotReadOnly, QueryStaleReadOnly, + ArrowCompressionCodecType, + ArrowCompressionCodec, + ArrowFormatSettings, + ArrowFormatMeta, ) from .pool import QuerySessionPool diff --git a/contrib/python/ydb/py3/ydb/query/base.py b/contrib/python/ydb/py3/ydb/query/base.py index d69aa81db08..524db6e4690 100644 --- a/contrib/python/ydb/py3/ydb/query/base.py +++ b/contrib/python/ydb/py3/ydb/query/base.py @@ -12,6 +12,7 @@ from typing import ( from .._grpc.grpcwrapper import ydb_query from .._grpc.grpcwrapper.ydb_query_public_types import ( BaseQueryTxMode, + ArrowFormatSettings, ) from ..connection import _RpcState as RpcState from .. import convert @@ -55,6 +56,18 @@ class QueryStatsMode(enum.IntEnum): PROFILE = 40 +class QuerySchemaInclusionMode(enum.IntEnum): + UNSPECIFIED = 0 + ALWAYS = 1 + FIRST_ONLY = 2 + + +class QueryResultSetFormat(enum.IntEnum): + UNSPECIFIED = 0 + VALUE = 1 + ARROW = 2 + + class SyncResponseContextIterator(_utilities.SyncResponseIterator): def __enter__(self) -> "SyncResponseContextIterator": return self @@ -139,6 +152,9 @@ def create_execute_query_request( syntax: Optional[QuerySyntax], exec_mode: Optional[QueryExecMode], stats_mode: Optional[QueryStatsMode], + schema_inclusion_mode: Optional[QuerySchemaInclusionMode], + result_set_format: Optional[QueryResultSetFormat], + arrow_format_settings: Optional[ArrowFormatSettings], parameters: Optional[dict], concurrent_result_sets: Optional[bool], ) -> ydb_query.ExecuteQueryRequest: @@ -146,6 +162,10 @@ def create_execute_query_request( syntax = QuerySyntax.YQL_V1 if not syntax else syntax exec_mode = QueryExecMode.EXECUTE if not exec_mode else exec_mode stats_mode = QueryStatsMode.NONE if stats_mode is None else stats_mode + schema_inclusion_mode = ( + QuerySchemaInclusionMode.ALWAYS if schema_inclusion_mode is None else schema_inclusion_mode + ) + result_set_format = QueryResultSetFormat.VALUE if result_set_format is None else result_set_format tx_control = None if not tx_id and not tx_mode: @@ -176,6 +196,9 @@ def create_execute_query_request( parameters=parameters, concurrent_result_sets=concurrent_result_sets, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, ) except BaseException as e: raise issues.ClientInternalError("Unable to prepare execute request") from e diff --git a/contrib/python/ydb/py3/ydb/query/session.py b/contrib/python/ydb/py3/ydb/query/session.py index 48fa6463b32..3c0f7bc3234 100644 --- a/contrib/python/ydb/py3/ydb/query/session.py +++ b/contrib/python/ydb/py3/ydb/query/session.py @@ -203,6 +203,9 @@ class BaseQuerySession: syntax: base.QuerySyntax = None, exec_mode: base.QueryExecMode = None, stats_mode: Optional[base.QueryStatsMode] = None, + schema_inclusion_mode: Optional[base.QuerySchemaInclusionMode] = None, + result_set_format: Optional[base.QueryResultSetFormat] = None, + arrow_format_settings: Optional[base.ArrowFormatSettings] = None, concurrent_result_sets: bool = False, settings: Optional[BaseRequestSettings] = None, ) -> Iterable[_apis.ydb_query.ExecuteQueryResponsePart]: @@ -218,6 +221,9 @@ class BaseQuerySession: syntax=syntax, exec_mode=exec_mode, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, concurrent_result_sets=concurrent_result_sets, ) @@ -336,6 +342,9 @@ class QuerySession(BaseQuerySession): settings: Optional[BaseRequestSettings] = None, *, stats_mode: Optional[base.QueryStatsMode] = None, + schema_inclusion_mode: Optional[base.QuerySchemaInclusionMode] = None, + result_set_format: Optional[base.QueryResultSetFormat] = None, + arrow_format_settings: Optional[base.ArrowFormatSettings] = None, ) -> base.SyncResponseContextIterator: """Sends a query to Query Service @@ -350,6 +359,13 @@ class QuerySession(BaseQuerySession): 2) QueryStatsMode.BASIC; 3) QueryStatsMode.FULL; 4) QueryStatsMode.PROFILE; + :param schema_inclusion_mode: Schema inclusion mode for result sets: + 1) QuerySchemaInclusionMode.ALWAYS, which is default; + 2) QuerySchemaInclusionMode.FIRST_ONLY. + :param result_set_format: Format of the result sets: + 1) QueryResultSetFormat.VALUE, which is default; + 2) QueryResultSetFormat.ARROW. + :param arrow_format_settings: Settings for Arrow format when result_set_format is ARROW. :return: Iterator with result sets """ @@ -362,6 +378,9 @@ class QuerySession(BaseQuerySession): syntax=syntax, exec_mode=exec_mode, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, concurrent_result_sets=concurrent_result_sets, settings=settings, ) diff --git a/contrib/python/ydb/py3/ydb/query/transaction.py b/contrib/python/ydb/py3/ydb/query/transaction.py index 008ac7c404f..055542964d4 100644 --- a/contrib/python/ydb/py3/ydb/query/transaction.py +++ b/contrib/python/ydb/py3/ydb/query/transaction.py @@ -293,6 +293,9 @@ class BaseQueryTxContext(base.CallbackHandler): syntax: Optional[base.QuerySyntax], exec_mode: Optional[base.QueryExecMode], stats_mode: Optional[base.QueryStatsMode], + schema_inclusion_mode: Optional[base.QuerySchemaInclusionMode], + result_set_format: Optional[base.QueryResultSetFormat], + arrow_format_settings: Optional[base.ArrowFormatSettings], concurrent_result_sets: Optional[bool], settings: Optional[BaseRequestSettings], ) -> Iterable[_apis.ydb_query.ExecuteQueryResponsePart]: @@ -311,6 +314,9 @@ class BaseQueryTxContext(base.CallbackHandler): syntax=syntax, exec_mode=exec_mode, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, concurrent_result_sets=concurrent_result_sets, ) @@ -462,6 +468,9 @@ class QueryTxContext(BaseQueryTxContext): settings: Optional[BaseRequestSettings] = None, *, stats_mode: Optional[base.QueryStatsMode] = None, + schema_inclusion_mode: Optional[base.QuerySchemaInclusionMode] = None, + result_set_format: Optional[base.QueryResultSetFormat] = None, + arrow_format_settings: Optional[base.ArrowFormatSettings] = None, ) -> base.SyncResponseContextIterator: """Sends a query to Query Service @@ -483,6 +492,13 @@ class QueryTxContext(BaseQueryTxContext): 2) QueryStatsMode.BASIC; 3) QueryStatsMode.FULL; 4) QueryStatsMode.PROFILE; + :param schema_inclusion_mode: Schema inclusion mode for result sets: + 1) QuerySchemaInclusionMode.ALWAYS, which is default; + 2) QuerySchemaInclusionMode.FIRST_ONLY. + :param result_set_format: Format of the result sets: + 1) QueryResultSetFormat.VALUE, which is default; + 2) QueryResultSetFormat.ARROW. + :param arrow_format_settings: Settings for Arrow format when result_set_format is ARROW. :return: Iterator with result sets """ @@ -494,6 +510,9 @@ class QueryTxContext(BaseQueryTxContext): syntax=syntax, exec_mode=exec_mode, stats_mode=stats_mode, + schema_inclusion_mode=schema_inclusion_mode, + result_set_format=result_set_format, + arrow_format_settings=arrow_format_settings, parameters=parameters, concurrent_result_sets=concurrent_result_sets, settings=settings, diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py index cd97a8555f7..5f976da1e51 100644 --- a/contrib/python/ydb/py3/ydb/ydb_version.py +++ b/contrib/python/ydb/py3/ydb/ydb_version.py @@ -1 +1 @@ -VERSION = "3.21.13" +VERSION = "3.22.0" |
