Intermediate changes

commit_hash:ff119d5d465aa66bb60906ff86071695dcd210e3

13 files changed, 58 insertions, 41 deletions

diff --git a/contrib/python/aiosignal/.dist-info/METADATA b/contrib/python/aiosignal/.dist-info/METADATA
index fc964525f05..6839bf9c50f 100644
--- a/contrib/python/aiosignal/.dist-info/METADATA
+++ b/contrib/python/aiosignal/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aiosignal
-Version: 1.3.1
+Version: 1.3.2
 Summary: aiosignal: a list of registered asynchronous callbacks
 Home-page: https://github.com/aio-libs/aiosignal
 Maintainer: aiohttp team <[email protected]>
@@ -17,20 +17,15 @@ Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Operating System :: POSIX
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: frozenlist (>=1.1.0)
+Requires-Dist: frozenlist>=1.1.0
 
 =========
 aiosignal
@@ -91,7 +86,7 @@ Installation
 
    $ pip install aiosignal
 
-The library requires Python 3.6 or newer.
+The library requires Python 3.8 or newer.
 
 
 Documentation
@@ -107,7 +102,7 @@ Communication channels
 Requirements
 ============
 
-- Python >= 3.6
+- Python >= 3.8
 - frozenlist >= 1.0.0
 
 License
diff --git a/contrib/python/aiosignal/README.rst b/contrib/python/aiosignal/README.rst
index d21fc96bd68..bc272d237ee 100644
--- a/contrib/python/aiosignal/README.rst
+++ b/contrib/python/aiosignal/README.rst
@@ -57,7 +57,7 @@ Installation
 
    $ pip install aiosignal
 
-The library requires Python 3.6 or newer.
+The library requires Python 3.8 or newer.
 
 
 Documentation
@@ -73,7 +73,7 @@ Communication channels
 Requirements
 ============
 
-- Python >= 3.6
+- Python >= 3.8
 - frozenlist >= 1.0.0
 
 License
diff --git a/contrib/python/aiosignal/aiosignal/__init__.py b/contrib/python/aiosignal/aiosignal/__init__.py
index 3d288e6ede6..4ad0278993a 100644
--- a/contrib/python/aiosignal/aiosignal/__init__.py
+++ b/contrib/python/aiosignal/aiosignal/__init__.py
@@ -1,6 +1,6 @@
 from frozenlist import FrozenList
 
-__version__ = "1.3.1"
+__version__ = "1.3.2"
 
 __all__ = ("Signal",)
 
diff --git a/contrib/python/aiosignal/ya.make b/contrib/python/aiosignal/ya.make
index 6094942c669..fa453a52fe7 100644
--- a/contrib/python/aiosignal/ya.make
+++ b/contrib/python/aiosignal/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(1.3.1)
+VERSION(1.3.2)
 
 LICENSE(Apache-2.0)
 
diff --git a/contrib/python/clickhouse-connect/.dist-info/METADATA b/contrib/python/clickhouse-connect/.dist-info/METADATA
index 774b5caa2e5..a8378b82544 100644
--- a/contrib/python/clickhouse-connect/.dist-info/METADATA
+++ b/contrib/python/clickhouse-connect/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clickhouse-connect
-Version: 0.8.9
+Version: 0.8.10
 Summary: ClickHouse Database Core Driver for Python, Pandas, and Superset
 Home-page: https://github.com/ClickHouse/clickhouse-connect
 Author: ClickHouse Inc.
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py b/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py
index eed7e58e362..c0bfba807c0 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py
@@ -1 +1 @@
-version = '0.8.9'
+version = '0.8.10'
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py
index e32145f1f87..6222400154c 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/datatypes/dynamic.py
@@ -2,7 +2,7 @@ from typing import List, Sequence, Collection
 
 from clickhouse_connect.datatypes.base import ClickHouseType, TypeDef
 from clickhouse_connect.datatypes.registry import get_from_name
-from clickhouse_connect.driver.common import unescape_identifier, first_value
+from clickhouse_connect.driver.common import unescape_identifier, first_value, write_uint64
 from clickhouse_connect.driver.ctypes import data_conv
 from clickhouse_connect.driver.errors import handle_error
 from clickhouse_connect.driver.exceptions import DataError
@@ -14,6 +14,8 @@ from clickhouse_connect.json_impl import any_to_json
 SHARED_DATA_TYPE: ClickHouseType
 STRING_DATA_TYPE: ClickHouseType
 
+json_serialization_format = 0x1
+
 class Variant(ClickHouseType):
     _slots = 'element_types'
     python_type = object
@@ -86,9 +88,11 @@ class Dynamic(ClickHouseType):
 
 
 def read_dynamic_prefix(source: ByteSource) -> List[ClickHouseType]:
-    if source.read_uint64() != 1:  # dynamic structure serialization version, currently only 1 is recognized
+    serialize_version = source.read_uint64()
+    if serialize_version == 1:
+        source.read_leb128()  # max dynamic types, we ignore this value
+    elif serialize_version != 2:
         raise DataError('Unrecognized dynamic structure version')
-    source.read_leb128()  # max dynamic types, we ignore this value
     num_variants = source.read_leb128()
     variant_types = [get_from_name(source.read_leb128_str()) for _ in range(num_variants)]
     variant_types.append(STRING_DATA_TYPE)
@@ -188,13 +192,23 @@ class JSON(ClickHouseType):
 
     @property
     def insert_name(self):
-        return 'String'
+        if json_serialization_format == 0:
+            return 'String'
+        return super().insert_name
+
+    def write_column_prefix(self, dest: bytearray):
+        if json_serialization_format > 0:
+            write_uint64(json_serialization_format, dest)
+
+    def read_column_prefix(self, source: ByteSource, ctx: QueryContext):
+        serialize_version = source.read_uint64()
+        if serialize_version == 0:
+            source.read_leb128()  # max dynamic types, we ignore this value
+        elif serialize_version != 2:
+            raise DataError(f'Unrecognized dynamic structure version: {serialize_version} column: `{ctx.column_name}`')
 
     # pylint: disable=too-many-locals
-    def read_column(self, source: ByteSource, num_rows: int, ctx: QueryContext):
-        if source.read_uint64() != 0: # object serialization version, currently only 0 is recognized
-            raise DataError(f'unrecognized object serialization version, column `{ctx.column_name}`')
-        source.read_leb128() # the max number of dynamic paths.  Used to preallocate storage in ClickHouse; we ignore it
+    def _read_column_binary(self, source: ByteSource, num_rows: int, ctx: QueryContext):
         dynamic_path_cnt = source.read_leb128()
         dynamic_paths = [source.read_leb128_str() for _ in range(dynamic_path_cnt)]
         for typed in self.typed_types:
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py
index b50b9bb6782..988cbf3aebf 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/buffer.py
@@ -129,10 +129,8 @@ class ResponseBuffer(ByteSource):
         return column
 
     @property
-    def last_message(self):
-        if len(self.buffer) == 0:
-            return None
-        return self.buffer.decode()
+    def last_message(self) -> bytes:
+        return self.buffer
 
     def close(self):
         if self.source:
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py
index 717a9093136..144f7b8913a 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py
@@ -12,6 +12,7 @@ from clickhouse_connect import common
 from clickhouse_connect.common import version
 from clickhouse_connect.datatypes.registry import get_from_name
 from clickhouse_connect.datatypes.base import ClickHouseType
+from clickhouse_connect.datatypes import dynamic as dynamic_module
 from clickhouse_connect.driver import tzutil
 from clickhouse_connect.driver.common import dict_copy, StreamContext, coerce_int, coerce_bool
 from clickhouse_connect.driver.constants import CH_VERSION_WITH_PROTOCOL, PROTOCOL_VERSION_WITH_LOW_CARD
@@ -90,7 +91,7 @@ class Client(ABC):
         server_settings = self.query(f'SELECT name, value, {readonly} as readonly FROM system.settings LIMIT 10000')
         self.server_settings = {row['name']: SettingDef(**row) for row in server_settings.named_results()}
 
-        if self.min_version(CH_VERSION_WITH_PROTOCOL):
+        if self.min_version(CH_VERSION_WITH_PROTOCOL) and common.get_setting('use_protocol_version'):
             #  Unfortunately we have to validate that the client protocol version is actually used by ClickHouse
             #  since the query parameter could be stripped off (in particular, by CHProxy)
             test_data = self.raw_query('SELECT 1 AS check', fmt='Native', settings={
@@ -103,6 +104,8 @@ class Client(ABC):
         if self._setting_status('allow_experimental_json_type').is_set and \
                 self._setting_status('cast_string_to_dynamic_user_inference').is_writable:
             self.set_client_setting('cast_string_to_dynamic_use_inference', '1')
+        if self.min_version('24.8') and not self.min_version('24.10'):
+            dynamic_module.json_serialization_format = 0
 
 
     def _validate_settings(self, settings: Optional[Dict[str, Any]]) -> Dict[str, str]:
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py
index 34ec1b7f4de..b5ae795c9ad 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/transform.py
@@ -55,13 +55,7 @@ class NativeTransform:
                     # We ran out of data before it was expected, this could be ClickHouse reporting an error
                     # in the response
                     if source.last_message:
-                        message = source.last_message
-                        if len(message) > 1024:
-                            message = message[-1024:]
-                        error_start = message.find('Code: ')
-                        if error_start != -1:
-                            message = message[error_start:]
-                        raise StreamFailureError(message) from None
+                        raise StreamFailureError(extract_error_message(source.last_message)) from None
                 raise
             block_num += 1
             return result_block
@@ -119,3 +113,16 @@ class NativeTransform:
                 yield footer
 
         return chunk_gen()
+
+
+def extract_error_message(message: bytes) -> str:
+    if len(message) > 1024:
+        message = message[-1024:]
+    error_start = message.find('Code: '.encode())
+    if error_start != -1:
+        message = message[error_start:]
+    try:
+        message_str = message.decode()
+    except UnicodeError:
+        message_str = f'unrecognized data found in stream: `{message.hex()[128:]}`'
+    return message_str
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py
index 015e162fbea..6a425e5e1f5 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/types.py
@@ -11,7 +11,7 @@ class Closable(ABC):
 
 
 class ByteSource(Closable):
-    last_message = None
+    last_message:bytes = None
 
     @abstractmethod
     def read_leb128(self) -> int:
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx
index 9e774cd5146..02b5514e0b4 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driverc/buffer.pyx
@@ -186,7 +186,7 @@ cdef class ResponseBuffer:
 
     @cython.boundscheck(False)
     @cython.wraparound(False)
-    def  read_byte(self) -> int:
+    def read_byte(self) -> int:
         if self.buf_loc < self.buf_sz:
             b = self.buffer[self.buf_loc]
             self.buf_loc += 1
@@ -299,8 +299,8 @@ cdef class ResponseBuffer:
     @property
     def last_message(self):
         if self.buffer == NULL:
-            return None
-        return self.buffer[self.buf_sz:].decode()
+            return self.slice[0:]
+        return self.buffer[self.buf_sz:]
 
     def __dealloc__(self):
         self.close()
diff --git a/contrib/python/clickhouse-connect/ya.make b/contrib/python/clickhouse-connect/ya.make
index f2bef2bd2a1..3ff1aff7343 100644
--- a/contrib/python/clickhouse-connect/ya.make
+++ b/contrib/python/clickhouse-connect/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(0.8.9)
+VERSION(0.8.10)
 
 LICENSE(Apache-2.0)