Merge branch 'rightlib' into mergelibs-241019-1758

17 files changed, 102 insertions, 31 deletions

diff --git a/contrib/python/clickhouse-connect/.dist-info/METADATA b/contrib/python/clickhouse-connect/.dist-info/METADATA
index 24f7a78836..bb928a4bc2 100644
--- a/contrib/python/clickhouse-connect/.dist-info/METADATA
+++ b/contrib/python/clickhouse-connect/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clickhouse-connect
-Version: 0.8.1
+Version: 0.8.2
 Summary: ClickHouse Database Core Driver for Python, Pandas, and Superset
 Home-page: https://github.com/ClickHouse/clickhouse-connect
 Author: ClickHouse Inc.
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py b/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py
index 398cfc4c74..c400d68132 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/__version__.py
@@ -1 +1 @@
-version = '0.8.1'
+version = '0.8.2'
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py
index fe11c27883..d6b84885e4 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py
@@ -59,9 +59,15 @@ class Client(ABC):
         """
         self.query_limit = coerce_int(query_limit)
         self.query_retries = coerce_int(query_retries)
+        if database and not database == '__default__':
+            self.database = database
         if show_clickhouse_errors is not None:
             self.show_clickhouse_errors = coerce_bool(show_clickhouse_errors)
         self.server_host_name = server_host_name
+        self.uri = uri
+        self._init_common_settings(apply_server_timezone)
+
+    def _init_common_settings(self, apply_server_timezone:Optional[Union[str, bool]] ):
         self.server_tz, dst_safe = pytz.UTC, True
         self.server_version, server_tz = \
             tuple(self.command('SELECT version(), timezone()', use_database=False))
@@ -83,8 +89,7 @@ class Client(ABC):
             readonly = common.get_setting('readonly')
         server_settings = self.query(f'SELECT name, value, {readonly} as readonly FROM system.settings LIMIT 10000')
         self.server_settings = {row['name']: SettingDef(**row) for row in server_settings.named_results()}
-        if database and not database == '__default__':
-            self.database = database
+
         if self.min_version(CH_VERSION_WITH_PROTOCOL):
             #  Unfortunately we have to validate that the client protocol version is actually used by ClickHouse
             #  since the query parameter could be stripped off (in particular, by CHProxy)
@@ -95,7 +100,9 @@ class Client(ABC):
                 self.protocol_version = PROTOCOL_VERSION_WITH_LOW_CARD
         if self._setting_status('date_time_input_format').is_writable:
             self.set_client_setting('date_time_input_format', 'best_effort')
-        self.uri = uri
+        if self._setting_status('allow_experimental_json_type').is_set:
+            self.set_client_setting('cast_string_to_dynamic_use_inference', '1')
+
 
     def _validate_settings(self, settings: Optional[Dict[str, Any]]) -> Dict[str, str]:
         """
@@ -655,7 +662,8 @@ class Client(ABC):
                            settings=settings, context=context)
 
     def insert_arrow(self, table: str,
-                     arrow_table, database: str = None,
+                     arrow_table,
+                     database: str = None,
                      settings: Optional[Dict] = None) -> QuerySummary:
         """
         Insert a PyArrow table DataFrame into ClickHouse using raw Arrow format
@@ -666,7 +674,8 @@ class Client(ABC):
         :return: QuerySummary with summary information, throws exception if insert fails
         """
         full_table = table if '.' in table or not database else f'{database}.{table}'
-        column_names, insert_block = arrow_buffer(arrow_table)
+        compression = self.write_compression if self.write_compression in ('zstd', 'lz4') else None
+        column_names, insert_block = arrow_buffer(arrow_table, compression)
         return self.raw_insert(full_table, column_names, insert_block, settings, 'Arrow')
 
     def create_insert_context(self,
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py
index 58b5460a59..558d66f614 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httputil.py
@@ -244,7 +244,8 @@ class ResponseSource:
                 else:
                     chunk = chunks.popleft()
                     current_size -= len(chunk)
-                yield chunk
+                if chunk:
+                    yield chunk
 
         self.gen = buffered()
 
diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py
index 54edbeff09..bd10270e71 100644
--- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py
+++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py
@@ -374,9 +374,12 @@ def to_arrow_batches(buffer: IOBase) -> StreamContext:
     return StreamContext(buffer, reader)
 
 
-def arrow_buffer(table) -> Tuple[Sequence[str], bytes]:
+def arrow_buffer(table, compression: Optional[str] = None) -> Tuple[Sequence[str], bytes]:
     pyarrow = check_arrow()
+    options = None
+    if compression in ('zstd', 'lz4'):
+        options = pyarrow.ipc.IpcWriteOptions(compression=pyarrow.Codec(compression=compression))
     sink = pyarrow.BufferOutputStream()
-    with pyarrow.RecordBatchFileWriter(sink, table.schema) as writer:
+    with pyarrow.RecordBatchFileWriter(sink, table.schema, options=options) as writer:
         writer.write(table)
     return table.schema.names, sink.getvalue()
diff --git a/contrib/python/clickhouse-connect/ya.make b/contrib/python/clickhouse-connect/ya.make
index e594301105..89d942df53 100644
--- a/contrib/python/clickhouse-connect/ya.make
+++ b/contrib/python/clickhouse-connect/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(0.8.1)
+VERSION(0.8.2)
 
 LICENSE(Apache-2.0)
 
diff --git a/contrib/python/ipython/py3/.dist-info/METADATA b/contrib/python/ipython/py3/.dist-info/METADATA
index db0f29ac01..b3a405d51d 100644
--- a/contrib/python/ipython/py3/.dist-info/METADATA
+++ b/contrib/python/ipython/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipython
-Version: 8.27.0
+Version: 8.28.0
 Summary: IPython: Productive Interactive Computing
 Author: The IPython Development Team
 Author-email: ipython-dev@python.org
diff --git a/contrib/python/ipython/py3/IPython/core/debugger.py b/contrib/python/ipython/py3/IPython/core/debugger.py
index a858972bb5..e7a0b8fb55 100644
--- a/contrib/python/ipython/py3/IPython/core/debugger.py
+++ b/contrib/python/ipython/py3/IPython/core/debugger.py
@@ -14,14 +14,35 @@ Among other things, this subclass of PDB:
  - hide frames in tracebacks based on `__tracebackhide__`
  - allows to skip frames based on `__debuggerskip__`
 
+
+Global Configuration
+--------------------
+
+The IPython debugger will by read the global ``~/.pdbrc`` file.
+That is to say you can list all comands supported by ipdb in your `~/.pdbrc`
+configuration file, to globally configure pdb.
+
+Example::
+
+   # ~/.pdbrc
+   skip_predicates debuggerskip false
+   skip_hidden false
+   context 25
+
+Features
+--------
+
+The IPython debugger can hide and skip frames when printing or moving through
+the stack. This can have a performance impact, so can be configures.
+
 The skipping and hiding frames are configurable via the `skip_predicates`
 command.
 
 By default, frames from readonly files will be hidden, frames containing
-``__tracebackhide__=True`` will be hidden.
+``__tracebackhide__ = True`` will be hidden.
 
-Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent
-frames value of ``__debuggerskip__`` is ``True`` will be skipped.
+Frames containing ``__debuggerskip__`` will be stepped over, frames whose parent
+frames value of ``__debuggerskip__`` is ``True`` will also be skipped.
 
     >>> def helpers_helper():
     ...     pass
@@ -1070,7 +1091,9 @@ class Pdb(OldPdb):
                 raise ValueError()
             self.context = new_context
         except ValueError:
-            self.error("The 'context' command requires a positive integer argument.")
+            self.error(
+                f"The 'context' command requires a positive integer argument (current value {self.context})."
+            )
 
 
 class InterruptiblePdb(Pdb):
diff --git a/contrib/python/ipython/py3/IPython/core/display.py b/contrib/python/ipython/py3/IPython/core/display.py
index 20e2e34b8f..5c4557b150 100644
--- a/contrib/python/ipython/py3/IPython/core/display.py
+++ b/contrib/python/ipython/py3/IPython/core/display.py
@@ -41,7 +41,11 @@ from warnings import warn
 
 def __getattr__(name):
     if name in _deprecated_names:
-        warn(f"Importing {name} from IPython.core.display is deprecated since IPython 7.14, please import from IPython display", DeprecationWarning, stacklevel=2)
+        warn(
+            f"Importing {name} from IPython.core.display is deprecated since IPython 7.14, please import from IPython.display",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         return getattr(display_functions, name)
 
     if name in globals().keys():
diff --git a/contrib/python/ipython/py3/IPython/core/magics/packaging.py b/contrib/python/ipython/py3/IPython/core/magics/packaging.py
index 093b0a2ec1..09d4117270 100644
--- a/contrib/python/ipython/py3/IPython/core/magics/packaging.py
+++ b/contrib/python/ipython/py3/IPython/core/magics/packaging.py
@@ -9,6 +9,7 @@
 #-----------------------------------------------------------------------------
 
 import functools
+import os
 import re
 import shlex
 import sys
@@ -41,6 +42,16 @@ def _get_conda_like_executable(command):
     executable: string
         Value should be: conda, mamba or micromamba
     """
+    # Check for a environment variable bound to the base executable, both conda and mamba
+    # set these when activating an environment.
+    base_executable = "CONDA_EXE"
+    if "mamba" in command.lower():
+        base_executable = "MAMBA_EXE"
+    if base_executable in os.environ:
+        executable = Path(os.environ[base_executable])
+        if executable.is_file():
+            return str(executable.resolve())
+
     # Check if there is a conda executable in the same directory as the Python executable.
     # This is the case within conda's root environment.
     executable = Path(sys.executable).parent / command
@@ -48,10 +59,12 @@ def _get_conda_like_executable(command):
         return str(executable)
 
     # Otherwise, attempt to extract the executable from conda history.
-    # This applies in any conda environment.
+    # This applies in any conda environment. Parsing this way is error prone because
+    # different versions of conda and mamba include differing cmd values such as
+    # `conda`, `conda-script.py`, or `path/to/conda`, here use the raw command provided.
     history = Path(sys.prefix, "conda-meta", "history").read_text(encoding="utf-8")
     match = re.search(
-        rf"^#\s*cmd:\s*(?P<command>.*{executable})\s[create|install]",
+        rf"^#\s*cmd:\s*(?P<command>.*{command})\s[create|install]",
         history,
         flags=re.MULTILINE,
     )
diff --git a/contrib/python/ipython/py3/IPython/core/release.py b/contrib/python/ipython/py3/IPython/core/release.py
index c77f561096..fb5a54da6a 100644
--- a/contrib/python/ipython/py3/IPython/core/release.py
+++ b/contrib/python/ipython/py3/IPython/core/release.py
@@ -16,7 +16,7 @@
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 8
-_version_minor = 27
+_version_minor = 28
 _version_patch = 0
 _version_extra = ".dev"
 # _version_extra = "rc1"
diff --git a/contrib/python/ipython/py3/IPython/core/ultratb.py b/contrib/python/ipython/py3/IPython/core/ultratb.py
index 15c835f4d2..e38ef29c7d 100644
--- a/contrib/python/ipython/py3/IPython/core/ultratb.py
+++ b/contrib/python/ipython/py3/IPython/core/ultratb.py
@@ -830,8 +830,8 @@ class VerboseTB(TBTools):
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
-    _tb_highlight = ""
-    _tb_highlight_style = "default"
+    tb_highlight = ""
+    tb_highlight_style = "default"
 
     def __init__(
         self,
@@ -1133,8 +1133,8 @@ class VerboseTB(TBTools):
         after = context // 2
         before = context - after
         if self.has_colors:
-            style = get_style_by_name(self._tb_highlight_style)
-            style = stack_data.style_with_executing_node(style, self._tb_highlight)
+            style = get_style_by_name(self.tb_highlight_style)
+            style = stack_data.style_with_executing_node(style, self.tb_highlight)
             formatter = Terminal256Formatter(style=style)
         else:
             formatter = None
diff --git a/contrib/python/ipython/py3/IPython/external/qt_loaders.py b/contrib/python/ipython/py3/IPython/external/qt_loaders.py
index 1486cf9d77..6058ee5a9a 100644
--- a/contrib/python/ipython/py3/IPython/external/qt_loaders.py
+++ b/contrib/python/ipython/py3/IPython/external/qt_loaders.py
@@ -302,13 +302,25 @@ def import_pyside6():
 
     ImportErrors raised within this function are non-recoverable
     """
+
+    def get_attrs(module):
+        return {
+            name: getattr(module, name)
+            for name in dir(module)
+            if not name.startswith("_")
+        }
+
     from PySide6 import QtGui, QtCore, QtSvg, QtWidgets, QtPrintSupport
 
     # Join QtGui and QtWidgets for Qt4 compatibility.
     QtGuiCompat = types.ModuleType("QtGuiCompat")
     QtGuiCompat.__dict__.update(QtGui.__dict__)
-    QtGuiCompat.__dict__.update(QtWidgets.__dict__)
-    QtGuiCompat.__dict__.update(QtPrintSupport.__dict__)
+    if QtCore.__version_info__ < (6, 7):
+        QtGuiCompat.__dict__.update(QtWidgets.__dict__)
+        QtGuiCompat.__dict__.update(QtPrintSupport.__dict__)
+    else:
+        QtGuiCompat.__dict__.update(get_attrs(QtWidgets))
+        QtGuiCompat.__dict__.update(get_attrs(QtPrintSupport))
 
     return QtCore, QtGuiCompat, QtSvg, QT_API_PYSIDE6
 
diff --git a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py
index 58edebdad2..304813b0f5 100644
--- a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py
+++ b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py
@@ -1,2 +1,2 @@
 # GENERATED BY setup.py
-commit = "82690a067"
+commit = "a9c7369d7"
diff --git a/contrib/python/ipython/py3/IPython/utils/terminal.py b/contrib/python/ipython/py3/IPython/utils/terminal.py
index b09cfe0d22..10d73fce58 100644
--- a/contrib/python/ipython/py3/IPython/utils/terminal.py
+++ b/contrib/python/ipython/py3/IPython/utils/terminal.py
@@ -80,7 +80,13 @@ def _set_term_title_xterm(title):
 def _restore_term_title_xterm():
     # Make sure the restore has at least one accompanying set.
     global _xterm_term_title_saved
-    assert _xterm_term_title_saved
+    if not _xterm_term_title_saved:
+        warnings.warn(
+            "Expecting xterm_term_title_saved to be True, but is not; will not restore terminal title.",
+            stacklevel=1,
+        )
+        return
+
     sys.stdout.write('\033[23;0t') 
     _xterm_term_title_saved = False
 
diff --git a/contrib/python/ipython/py3/patches/03-dissable-backgroud-highlighting.patch b/contrib/python/ipython/py3/patches/03-dissable-backgroud-highlighting.patch
index 73b189b863..4694b5b2e8 100644
--- a/contrib/python/ipython/py3/patches/03-dissable-backgroud-highlighting.patch
+++ b/contrib/python/ipython/py3/patches/03-dissable-backgroud-highlighting.patch
@@ -1,5 +1,5 @@
 --- contrib/python/ipython/py3/IPython/core/ultratb.py	(index)
 +++ contrib/python/ipython/py3/IPython/core/ultratb.py	(working tree)
 @@ -613,1 +613,1 @@ class VerboseTB(TBTools):
--    _tb_highlight = "bg:ansiyellow"
-+    _tb_highlight = ""
+-    tb_highlight = "bg:ansiyellow"
++    tb_highlight = ""
diff --git a/contrib/python/ipython/py3/ya.make b/contrib/python/ipython/py3/ya.make
index ab2a7fb8f5..b13b8d1baa 100644
--- a/contrib/python/ipython/py3/ya.make
+++ b/contrib/python/ipython/py3/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(8.27.0)
+VERSION(8.28.0)
 
 LICENSE(BSD-3-Clause)