aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2024-03-08 12:14:09 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2024-03-08 12:22:31 +0300
commit9584fd808b5bbcd989348a67768064134e3341d9 (patch)
tree601ec8deabb8dcdca66d4123cd7a0dcf7d804243
parente8116f70daae60dcbf13132581dba893802ce3fe (diff)
downloadydb-9584fd808b5bbcd989348a67768064134e3341d9.tar.gz
Intermediate changes
-rw-r--r--contrib/python/hypothesis/py3/.dist-info/METADATA2
-rw-r--r--contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py11
-rw-r--r--contrib/python/hypothesis/py3/hypothesis/stateful.py19
-rw-r--r--contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py35
-rw-r--r--contrib/python/hypothesis/py3/hypothesis/version.py2
-rw-r--r--contrib/python/hypothesis/py3/ya.make2
-rw-r--r--contrib/python/pg8000/.dist-info/METADATA1073
-rw-r--r--contrib/python/pg8000/README.rst1071
-rw-r--r--contrib/python/pg8000/pg8000/core.py10
-rw-r--r--contrib/python/pg8000/ya.make2
-rw-r--r--library/cpp/yt/misc/optional.h (renamed from yt/yt/core/misc/optional.h)0
-rw-r--r--yt/yt/core/concurrency/fair_share_action_queue-inl.h12
-rw-r--r--yt/yt/core/concurrency/fair_share_action_queue.cpp43
-rw-r--r--yt/yt/core/concurrency/fair_share_action_queue.h8
-rw-r--r--yt/yt/core/concurrency/fair_share_invoker_queue.cpp5
-rw-r--r--yt/yt/core/concurrency/fair_share_invoker_queue.h4
-rw-r--r--yt/yt/core/concurrency/invoker_queue.cpp67
-rw-r--r--yt/yt/core/concurrency/invoker_queue.h8
-rw-r--r--yt/yt/core/concurrency/profiling_helpers.cpp2
-rw-r--r--yt/yt/core/concurrency/profiling_helpers.h1
-rw-r--r--yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp166
-rw-r--r--yt/yt/core/concurrency/unittests/ya.make1
-rw-r--r--yt/yt/core/misc/common.h4
-rw-r--r--yt/yt/core/misc/protobuf_helpers.h2
-rw-r--r--yt/yt/core/yson/pull_parser-inl.h3
-rw-r--r--yt/yt/core/ytree/attributes.h2
-rw-r--r--yt/yt/core/ytree/yson_struct_detail.h3
27 files changed, 318 insertions, 2240 deletions
diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA
index d883f6dec6..171a09eebb 100644
--- a/contrib/python/hypothesis/py3/.dist-info/METADATA
+++ b/contrib/python/hypothesis/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: hypothesis
-Version: 6.98.9
+Version: 6.98.10
Summary: A library for property-based testing
Home-page: https://hypothesis.works
Author: David R. MacIver and Zac Hatfield-Dodds
diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py
index 0016f2fa1a..cdce1afacf 100644
--- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py
+++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py
@@ -40,7 +40,7 @@ import attr
from hypothesis.errors import Frozen, InvalidArgument, StopTest
from hypothesis.internal.cache import LRUReusedCache
-from hypothesis.internal.compat import floor, int_from_bytes, int_to_bytes
+from hypothesis.internal.compat import add_note, floor, int_from_bytes, int_to_bytes
from hypothesis.internal.conjecture.floats import float_to_lex, lex_to_float
from hypothesis.internal.conjecture.junkdrawer import IntList, uniform
from hypothesis.internal.conjecture.utils import (
@@ -1748,14 +1748,17 @@ class ConjectureData:
try:
if not at_top_level:
return strategy.do_draw(self)
- else:
- assert start_time is not None
+ assert start_time is not None
+ key = observe_as or f"generate:unlabeled_{len(self.draw_times)}"
+ try:
strategy.validate()
try:
return strategy.do_draw(self)
finally:
- key = observe_as or f"generate:unlabeled_{len(self.draw_times)}"
self.draw_times[key] = time.perf_counter() - start_time
+ except Exception as err:
+ add_note(err, f"while generating {key[9:]!r} from {strategy!r}")
+ raise
finally:
self.stop_example()
diff --git a/contrib/python/hypothesis/py3/hypothesis/stateful.py b/contrib/python/hypothesis/py3/hypothesis/stateful.py
index bf7271fc4c..2ab7ef13d5 100644
--- a/contrib/python/hypothesis/py3/hypothesis/stateful.py
+++ b/contrib/python/hypothesis/py3/hypothesis/stateful.py
@@ -47,6 +47,7 @@ from hypothesis._settings import (
from hypothesis.control import _current_build_context, current_build_context
from hypothesis.core import TestFunc, given
from hypothesis.errors import InvalidArgument, InvalidDefinition
+from hypothesis.internal.compat import add_note
from hypothesis.internal.conjecture import utils as cu
from hypothesis.internal.healthcheck import fail_health_check
from hypothesis.internal.observability import TESTCASE_CALLBACKS
@@ -439,18 +440,16 @@ class Rule:
bundles = attr.ib(init=False)
def __attrs_post_init__(self):
- arguments = {}
+ self.arguments_strategies = {}
bundles = []
for k, v in sorted(self.arguments.items()):
assert not isinstance(v, BundleReferenceStrategy)
if isinstance(v, Bundle):
bundles.append(v)
consume = isinstance(v, BundleConsumer)
- arguments[k] = BundleReferenceStrategy(v.name, consume=consume)
- else:
- arguments[k] = v
+ v = BundleReferenceStrategy(v.name, consume=consume)
+ self.arguments_strategies[k] = v
self.bundles = tuple(bundles)
- self.arguments_strategy = st.fixed_dictionaries(arguments)
self_strategy = st.runner()
@@ -978,7 +977,15 @@ class RuleStrategy(SearchStrategy):
.filter(lambda r: feature_flags.is_enabled(r.function.__name__))
)
- return (rule, data.draw(rule.arguments_strategy))
+ arguments = {}
+ for k, strat in rule.arguments_strategies.items():
+ try:
+ arguments[k] = data.draw(strat)
+ except Exception as err:
+ rname = rule.function.__name__
+ add_note(err, f"while generating {k!r} from {strat!r} for rule {rname}")
+ raise
+ return (rule, arguments)
def is_valid(self, rule):
predicates = self.machine._observability_predicates
diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py
index d91102cb50..7c4e584ee6 100644
--- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py
+++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py
@@ -37,6 +37,7 @@ from hypothesis.errors import HypothesisWarning, InvalidArgument, ResolutionFail
from hypothesis.internal.compat import PYPY, BaseExceptionGroup, ExceptionGroup
from hypothesis.internal.conjecture.utils import many as conjecture_utils_many
from hypothesis.internal.filtering import max_len, min_len
+from hypothesis.internal.reflection import get_pretty_function_description
from hypothesis.strategies._internal.datetime import zoneinfo # type: ignore
from hypothesis.strategies._internal.ipaddress import (
SPECIAL_IPv4_RANGES,
@@ -300,16 +301,40 @@ def _get_constraints(args: Tuple[Any, ...]) -> Iterator["at.BaseMetadata"]:
yield from at.Len(arg.start or 0, arg.stop)
+def _flat_annotated_repr_parts(annotated_type):
+ # Helper to get a good error message in find_annotated_strategy() below.
+ type_reps = [
+ get_pretty_function_description(a)
+ for a in annotated_type.__args__
+ if not isinstance(a, typing.TypeVar)
+ ]
+ metadata_reps = []
+ for m in getattr(annotated_type, "__metadata__", ()):
+ if is_annotated_type(m):
+ ts, ms = _flat_annotated_repr_parts(m)
+ type_reps.extend(ts)
+ metadata_reps.extend(ms)
+ else:
+ metadata_reps.append(get_pretty_function_description(m))
+ return type_reps, metadata_reps
+
+
def find_annotated_strategy(annotated_type):
metadata = getattr(annotated_type, "__metadata__", ())
if any(is_annotated_type(arg) for arg in metadata):
- # We are in the case where one of the metadata argument
- # is itself an annotated type. Although supported at runtime,
- # This shouldn't be allowed: we prefer to raise here
+ # Annotated[Annotated[T], ...] is perfectly acceptable, but it's all to easy
+ # to instead write Annotated[T1, Annotated[T2, ...]] - and nobody else checks
+ # for that at runtime. Once you add generics this can be seriously confusing,
+ # so we go to some trouble to give a helpful error message.
+ # For details: https://github.com/HypothesisWorks/hypothesis/issues/3891
+ ty_rep = repr(annotated_type).replace("typing.Annotated", "Annotated")
+ ts, ms = _flat_annotated_repr_parts(annotated_type)
+ bits = ", ".join([" | ".join(dict.fromkeys(ts or "?")), *dict.fromkeys(ms)])
raise ResolutionFailed(
- f"Failed to resolve strategy for the following Annotated type: {annotated_type}."
- "Arguments to the Annotated type cannot be Annotated."
+ f"`{ty_rep}` is invalid because nesting Annotated is only allowed for "
+ f"the first (type) argument, not for later (metadata) arguments. "
+ f"Did you mean `Annotated[{bits}]`?"
)
for arg in reversed(metadata):
if isinstance(arg, st.SearchStrategy):
diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py
index f54f8dc42f..ce617c47e9 100644
--- a/contrib/python/hypothesis/py3/hypothesis/version.py
+++ b/contrib/python/hypothesis/py3/hypothesis/version.py
@@ -8,5 +8,5 @@
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.
-__version_info__ = (6, 98, 9)
+__version_info__ = (6, 98, 10)
__version__ = ".".join(map(str, __version_info__))
diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make
index 89269430da..4c7de0c00e 100644
--- a/contrib/python/hypothesis/py3/ya.make
+++ b/contrib/python/hypothesis/py3/ya.make
@@ -2,7 +2,7 @@
PY3_LIBRARY()
-VERSION(6.98.9)
+VERSION(6.98.10)
LICENSE(MPL-2.0)
diff --git a/contrib/python/pg8000/.dist-info/METADATA b/contrib/python/pg8000/.dist-info/METADATA
index 93aec222db..291f74df6f 100644
--- a/contrib/python/pg8000/.dist-info/METADATA
+++ b/contrib/python/pg8000/.dist-info/METADATA
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: pg8000
-Version: 1.30.4
+Version: 1.30.5
Summary: PostgreSQL interface library
License: BSD 3-Clause License
Project-URL: Homepage, https://github.com/tlocke/pg8000
@@ -1351,839 +1351,13 @@ pg8000 encounters a network problem it'll raise an ``InterfaceError`` with the m
Native API Docs
---------------
-pg8000.native.Error
-```````````````````
-
-Generic exception that is the base exception of the other error exceptions.
-
-
-pg8000.native.InterfaceError
-````````````````````````````
-
-For errors that originate within pg8000.
-
-
-pg8000.native.DatabaseError
-```````````````````````````
-
-For errors that originate from the server.
-
-pg8000.native.Connection(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None)
-``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````
-
-Creates a connection to a PostgreSQL database.
-
-user
- The username to connect to the PostgreSQL server with. If your server character
- encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes,
- eg. ``'my_name'.encode('EUC-JP')``.
-
-host
- The hostname of the PostgreSQL server to connect with. Providing this parameter is
- necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be
- provided. The default is ``localhost``.
-
-database
- The name of the database instance to connect with. If ``None`` then the PostgreSQL
- server will assume the database name is the same as the username. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database``
- as bytes, eg. ``'my_db'.encode('EUC-JP')``.
-
-port
- The TCP/IP port of the PostgreSQL server instance. This parameter defaults to
- ``5432``, the registered common port of PostgreSQL TCP/IP servers.
-
-password
- The user password to connect to the server with. This parameter is optional; if
- omitted and the database server requests password-based authentication, the connection
- will fail to open. If this parameter is provided but not
- requested by the server, no error will occur.
-
- If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``.
-
-source_address
- The source IP address which initiates the connection to the PostgreSQL server. The
- default is ``None`` which means that the operating system will choose the source
- address.
-
-unix_sock
- The path to the UNIX socket to access the database through, for example,
- ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided.
-
-ssl_context
- This governs SSL encryption for TCP/IP sockets. It can have three values:
-
- - ``None``, meaning no SSL (the default)
-
- - ``True``, means use SSL with an |ssl.SSLContext|_ created using
- |ssl.create_default_context()|_
-
- - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection.
-
- If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific
- attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL
- socket, but not to request SSL from the PostgreSQL server. Note that this means you
- can't use SCRAM authentication with channel binding.
-
-timeout
- This is the time in seconds before the connection to the server will time out. The
- default is ``None`` which means no timeout.
-
-tcp_keepalive
- If ``True`` then use `TCP keepalive
- <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``.
-
-application_name
- Sets the `application_name
- <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_.
- If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default
- is ``None`` which means that the server will set the application name.
-
-replication
- Used to run in `streaming replication mode
- <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide values as
- bytes, eg. ``'database'.encode('EUC-JP')``.
-
-sock
- A socket-like object to use for the connection. For example, ``sock`` could be a plain
- ``socket.socket``, or it could represent an SSH tunnel or perhaps an
- ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be
- used to attempt to create an SSL socket from the provided socket.
-
-pg8000.native.Connection.notifications
-``````````````````````````````````````
-
-A deque of server-side `notifications
-<https://www.postgresql.org/docs/current/sql-notify.html>`__ received by this database
-connection (via the ``LISTEN`` / ``NOTIFY`` PostgreSQL commands). Each list item is a
-three-element tuple containing the PostgreSQL backend PID that issued the notify, the
-channel and the payload.
-
-
-pg8000.native.Connection.notices
-````````````````````````````````
-
-A deque of server-side notices received by this database connection.
-
-
-pg8000.native.Connection.parameter_statuses
-```````````````````````````````````````````
-
-A deque of server-side parameter statuses received by this database connection.
-
-
-pg8000.native.Connection.run(sql, stream=None, types=None, \*\*kwargs)
-``````````````````````````````````````````````````````````````````````
-
-Executes an sql statement, and returns the results as a ``list``. For example::
-
- con.run("SELECT * FROM cities where population > :pop", pop=10000)
-
-sql
- The SQL statement to execute. Parameter placeholders appear as a ``:`` followed by the
- parameter name.
-
-stream
- For use with the PostgreSQL `COPY
- <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. The nature
- of the parameter depends on whether the SQL command is ``COPY FROM`` or ``COPY TO``.
-
- ``COPY FROM``
- The stream parameter must be a readable file-like object or an iterable. If it's an
- iterable then the items can be ``str`` or binary.
- ``COPY TO``
- The stream parameter must be a writable file-like object.
-
-types
- A dictionary of oids. A key corresponds to a parameter.
-
-kwargs
- The parameters of the SQL statement.
-
-
-pg8000.native.Connection.row_count
-``````````````````````````````````
-
-This read-only attribute contains the number of rows that the last ``run()`` method
-produced (for query statements like ``SELECT``) or affected (for modification statements
-like ``UPDATE``.
-
-The value is -1 if:
-
-- No ``run()`` method has been performed yet.
-- There was no rowcount associated with the last ``run()``.
-
-
-pg8000.native.Connection.columns
-````````````````````````````````
-
-A list of column metadata. Each item in the list is a dictionary with the following
-keys:
-
-- name
-- table_oid
-- column_attrnum
-- type_oid
-- type_size
-- type_modifier
-- format
-
-
-pg8000.native.Connection.close()
-````````````````````````````````
-
-Closes the database connection.
-
-
-pg8000.native.Connection.register_out_adapter(typ, out_func)
-````````````````````````````````````````````````````````````
-
-Register a type adapter for types going out from pg8000 to the server.
-
-typ
- The Python class that the adapter is for.
-
-out_func
- A function that takes the Python object and returns its string representation
- in the format that the server requires.
-
-
-pg8000.native.Connection.register_in_adapter(oid, in_func)
-``````````````````````````````````````````````````````````
-
-Register a type adapter for types coming in from the server to pg8000.
-
-oid
- The PostgreSQL type identifier found in the `pg_type system catalog
- <https://www.postgresql.org/docs/current/catalog-pg-type.html>`_.
-
-in_func
- A function that takes the PostgreSQL string representation and returns a corresponding
- Python object.
-
-
-pg8000.native.Connection.prepare(sql)
-`````````````````````````````````````
-
-Returns a ``PreparedStatement`` object which represents a `prepared statement
-<https://www.postgresql.org/docs/current/sql-prepare.html>`_ on the server. It can
-subsequently be repeatedly executed.
-
-sql
- The SQL statement to prepare. Parameter placeholders appear as a ``:`` followed by the
- parameter name.
-
-
-pg8000.native.PreparedStatement
-```````````````````````````````
-
-A prepared statement object is returned by the ``pg8000.native.Connection.prepare()``
-method of a connection. It has the following methods:
-
-
-pg8000.native.PreparedStatement.run(\*\*kwargs)
-```````````````````````````````````````````````
-
-Executes the prepared statement, and returns the results as a ``tuple``.
-
-kwargs
- The parameters of the prepared statement.
-
-
-pg8000.native.PreparedStatement.close()
-```````````````````````````````````````
-
-Closes the prepared statement, releasing the prepared statement held on the server.
-
-
-pg8000.native.identifier(ident)
-```````````````````````````````
-
-Correctly quotes and escapes a string to be used as an `SQL identifier
-<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS>`_.
-
-ident
- The ``str`` to be used as an SQL identifier.
-
-
-pg8000.native.literal(value)
-````````````````````````````
-
-Correctly quotes and escapes a value to be used as an `SQL literal
-<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS>`_.
-
-value
- The value to be used as an SQL literal.
+`Native API Docs <docs/native_api_docs.rst>`_
DB-API 2 Docs
-------------
-
-Properties
-``````````
-
-
-pg8000.dbapi.apilevel
-:::::::::::::::::::::
-
-The DBAPI level supported, currently "2.0".
-
-
-pg8000.dbapi.threadsafety
-:::::::::::::::::::::::::
-
-Integer constant stating the level of thread safety the DBAPI interface supports. For
-pg8000, the threadsafety value is 1, meaning that threads may share the module but not
-connections.
-
-
-pg8000.dbapi.paramstyle
-:::::::::::::::::::::::
-
-String property stating the type of parameter marker formatting expected by
-the interface. This value defaults to "format", in which parameters are
-marked in this format: "WHERE name=%s".
-
-As an extension to the DBAPI specification, this value is not constant; it can be
-changed to any of the following values:
-
-qmark
- Question mark style, eg. ``WHERE name=?``
-
-numeric
- Numeric positional style, eg. ``WHERE name=:1``
-
-named
- Named style, eg. ``WHERE name=:paramname``
-
-format
- printf format codes, eg. ``WHERE name=%s``
-
-pyformat
- Python format codes, eg. ``WHERE name=%(paramname)s``
-
-
-pg8000.dbapi.STRING
-:::::::::::::::::::
-
-String type oid.
-
-pg8000.dbapi.BINARY
-:::::::::::::::::::
-
-
-pg8000.dbapi.NUMBER
-:::::::::::::::::::
-
-Numeric type oid.
-
-
-pg8000.dbapi.DATETIME
-:::::::::::::::::::::
-
-Timestamp type oid
-
-
-pg8000.dbapi.ROWID
-::::::::::::::::::
-
-ROWID type oid
-
-
-Functions
-`````````
-
-pg8000.dbapi.connect(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None)
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-Creates a connection to a PostgreSQL database.
-
-user
- The username to connect to the PostgreSQL server with. If your server character
- encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes,
- eg. ``'my_name'.encode('EUC-JP')``.
-
-host
- The hostname of the PostgreSQL server to connect with. Providing this parameter is
- necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be
- provided. The default is ``localhost``.
-
-database
- The name of the database instance to connect with. If ``None`` then the PostgreSQL
- server will assume the database name is the same as the username. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database``
- as bytes, eg. ``'my_db'.encode('EUC-JP')``.
-
-port
- The TCP/IP port of the PostgreSQL server instance. This parameter defaults to
- ``5432``, the registered common port of PostgreSQL TCP/IP servers.
-
-password
- The user password to connect to the server with. This parameter is optional; if
- omitted and the database server requests password-based authentication, the
- connection will fail to open. If this parameter is provided but not requested by the
- server, no error will occur.
-
- If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``.
-
-source_address
- The source IP address which initiates the connection to the PostgreSQL server. The
- default is ``None`` which means that the operating system will choose the source
- address.
-
-unix_sock
- The path to the UNIX socket to access the database through, for example,
- ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided.
-
-ssl_context
- This governs SSL encryption for TCP/IP sockets. It can have three values:
-
- - ``None``, meaning no SSL (the default)
- - ``True``, means use SSL with an |ssl.SSLContext|_ created using
- |ssl.create_default_context()|_.
-
- - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection.
-
- If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific
- attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL
- socket, but not to request SSL from the PostgreSQL server. Note that this means you
- can't use SCRAM authentication with channel binding.
-
-timeout
- This is the time in seconds before the connection to the server will time out. The
- default is ``None`` which means no timeout.
-
-tcp_keepalive
- If ``True`` then use `TCP keepalive
- <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``.
-
-application_name
- Sets the `application_name
- <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_. If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default
- is ``None`` which means that the server will set the application name.
-
-replication
- Used to run in `streaming replication mode
- <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide values as
- bytes, eg. ``'database'.encode('EUC-JP')``.
-
-sock
- A socket-like object to use for the connection. For example, ``sock`` could be a plain
- ``socket.socket``, or it could represent an SSH tunnel or perhaps an
- ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be
- used to attempt to create an SSL socket from the provided socket.
-
-
-pg8000.dbapi.Date(year, month, day)
-
-Construct an object holding a date value.
-
-This property is part of the `DBAPI 2.0 specification
-<http://www.python.org/dev/peps/pep-0249/>`_.
-
-Returns: `datetime.date`
-
-
-pg8000.dbapi.Time(hour, minute, second)
-:::::::::::::::::::::::::::::::::::::::
-
-Construct an object holding a time value.
-
-Returns: ``datetime.time``
-
-
-pg8000.dbapi.Timestamp(year, month, day, hour, minute, second)
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-Construct an object holding a timestamp value.
-
-Returns: ``datetime.datetime``
-
-
-pg8000.dbapi.DateFromTicks(ticks)
-:::::::::::::::::::::::::::::::::
-
-Construct an object holding a date value from the given ticks value (number of seconds
-since the epoch).
-
-Returns: ``datetime.datetime``
-
-
-pg8000.dbapi.TimeFromTicks(ticks)
-:::::::::::::::::::::::::::::::::
-
-Construct an object holding a time value from the given ticks value (number of seconds
-since the epoch).
-
-Returns: ``datetime.time``
-
-
-pg8000.dbapi.TimestampFromTicks(ticks)
-::::::::::::::::::::::::::::::::::::::
-
-Construct an object holding a timestamp value from the given ticks value (number of
-seconds since the epoch).
-
-Returns: ``datetime.datetime``
-
-
-pg8000.dbapi.Binary(value)
-::::::::::::::::::::::::::
-
-Construct an object holding binary data.
-
-Returns: ``bytes``.
-
-
-Generic Exceptions
-``````````````````
-
-Pg8000 uses the standard DBAPI 2.0 exception tree as "generic" exceptions. Generally,
-more specific exception types are raised; these specific exception types are derived
-from the generic exceptions.
-
-pg8000.dbapi.Warning
-::::::::::::::::::::
-
-Generic exception raised for important database warnings like data truncations. This
-exception is not currently used by pg8000.
-
-
-pg8000.dbapi.Error
-::::::::::::::::::
-
-Generic exception that is the base exception of all other error exceptions.
-
-
-pg8000.dbapi.InterfaceError
-:::::::::::::::::::::::::::
-
-Generic exception raised for errors that are related to the database interface rather
-than the database itself. For example, if the interface attempts to use an SSL
-connection but the server refuses, an InterfaceError will be raised.
-
-
-pg8000.dbapi.DatabaseError
-::::::::::::::::::::::::::
-
-Generic exception raised for errors that are related to the database. This exception is
-currently never raised by pg8000.
-
-
-pg8000.dbapi.DataError
-::::::::::::::::::::::
-
-Generic exception raised for errors that are due to problems with the processed data.
-This exception is not currently raised by pg8000.
-
-
-pg8000.dbapi.OperationalError
-:::::::::::::::::::::::::::::
-
-Generic exception raised for errors that are related to the database's operation and not
-necessarily under the control of the programmer. This exception is currently never
-raised by pg8000.
-
-
-pg8000.dbapi.IntegrityError
-:::::::::::::::::::::::::::
-
-Generic exception raised when the relational integrity of the database is affected. This
-exception is not currently raised by pg8000.
-
-
-pg8000.dbapi.InternalError
-::::::::::::::::::::::::::
-
-Generic exception raised when the database encounters an internal error. This is
-currently only raised when unexpected state occurs in the pg8000 interface itself, and
-is typically the result of a interface bug.
-
-
-pg8000.dbapi.ProgrammingError
-:::::::::::::::::::::::::::::
-
-Generic exception raised for programming errors. For example, this exception is raised
-if more parameter fields are in a query string than there are available parameters.
-
-
-pg8000.dbapi.NotSupportedError
-::::::::::::::::::::::::::::::
-
-Generic exception raised in case a method or database API was used which is not
-supported by the database.
-
-
-Classes
-```````
-
-
-pg8000.dbapi.Connection
-:::::::::::::::::::::::
-
-A connection object is returned by the ``pg8000.connect()`` function. It represents a
-single physical connection to a PostgreSQL database.
-
-
-pg8000.dbapi.Connection.autocommit
-::::::::::::::::::::::::::::::::::
-
-Following the DB-API specification, autocommit is off by default. It can be turned on by
-setting this boolean pg8000-specific autocommit property to ``True``.
-
-
-pg8000.dbapi.Connection.close()
-:::::::::::::::::::::::::::::::
-
-Closes the database connection.
-
-
-pg8000.dbapi.Connection.cursor()
-::::::::::::::::::::::::::::::::
-
-Creates a ``pg8000.dbapi.Cursor`` object bound to this connection.
-
-
-pg8000.dbapi.Connection.rollback()
-::::::::::::::::::::::::::::::::::
-
-Rolls back the current database transaction.
-
-
-pg8000.dbapi.Connection.tpc_begin(xid)
-::::::::::::::::::::::::::::::::::::::
-
-Begins a TPC transaction with the given transaction ID xid. This method should be
-called outside of a transaction (i.e. nothing may have executed since the last
-``commit()`` or ``rollback()``. Furthermore, it is an error to call ``commit()`` or
-``rollback()`` within the TPC transaction. A ``ProgrammingError`` is raised, if the
-application calls ``commit()`` or ``rollback()`` during an active TPC transaction.
-
-
-pg8000.dbapi.Connection.tpc_commit(xid=None)
-::::::::::::::::::::::::::::::::::::::::::::
-
-When called with no arguments, ``tpc_commit()`` commits a TPC transaction previously
-prepared with ``tpc_prepare()``. If ``tpc_commit()`` is called prior to
-``tpc_prepare()``, a single phase commit is performed. A transaction manager may choose
-to do this if only a single resource is participating in the global transaction.
-
-When called with a transaction ID ``xid``, the database commits the given transaction.
-If an invalid transaction ID is provided, a ``ProgrammingError`` will be raised. This
-form should be called outside of a transaction, and is intended for use in recovery.
-
-On return, the TPC transaction is ended.
-
-
-pg8000.dbapi.Connection.tpc_prepare()
-:::::::::::::::::::::::::::::::::::::
-
-Performs the first phase of a transaction started with ``.tpc_begin()``. A
-``ProgrammingError`` is be raised if this method is called outside of a TPC transaction.
-
-After calling ``tpc_prepare()``, no statements can be executed until ``tpc_commit()`` or
-``tpc_rollback()`` have been called.
-
-
-pg8000.dbapi.Connection.tpc_recover()
-:::::::::::::::::::::::::::::::::::::
-
-Returns a list of pending transaction IDs suitable for use with ``tpc_commit(xid)`` or
-``tpc_rollback(xid)``.
-
-
-pg8000.dbapi.Connection.tpc_rollback(xid=None)
-::::::::::::::::::::::::::::::::::::::::::::::
-
-When called with no arguments, ``tpc_rollback()`` rolls back a TPC transaction. It may
-be called before or after ``tpc_prepare()``.
-
-When called with a transaction ID xid, it rolls back the given transaction. If an
-invalid transaction ID is provided, a ``ProgrammingError`` is raised. This form should
-be called outside of a transaction, and is intended for use in recovery.
-
-On return, the TPC transaction is ended.
-
-
-pg8000.dbapi.Connection.xid(format_id, global_transaction_id, branch_qualifier)
-:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-Create a Transaction IDs (only global_transaction_id is used in pg) format_id and
-branch_qualifier are not used in postgres global_transaction_id may be any string
-identifier supported by postgres returns a tuple (format_id, global_transaction_id,
-branch_qualifier)
-
-
-pg8000.dbapi.Cursor
-:::::::::::::::::::
-
-A cursor object is returned by the ``pg8000.dbapi.Connection.cursor()`` method of a
-connection. It has the following attributes and methods:
-
-pg8000.dbapi.Cursor.arraysize
-'''''''''''''''''''''''''''''
-
-This read/write attribute specifies the number of rows to fetch at a time with
-``pg8000.dbapi.Cursor.fetchmany()``. It defaults to 1.
-
-
-pg8000.dbapi.Cursor.connection
-''''''''''''''''''''''''''''''
-
-This read-only attribute contains a reference to the connection object (an instance of
-``pg8000.dbapi.Connection``) on which the cursor was created.
-
-
-pg8000.dbapi.Cursor.rowcount
-''''''''''''''''''''''''''''
-
-This read-only attribute contains the number of rows that the last ``execute()`` or
-``executemany()`` method produced (for query statements like ``SELECT``) or affected
-(for modification statements like ``UPDATE``.
-
-The value is -1 if:
-
-- No ``execute()`` or ``executemany()`` method has been performed yet on the cursor.
-
-- There was no rowcount associated with the last ``execute()``.
-
-- At least one of the statements executed as part of an ``executemany()`` had no row
- count associated with it.
-
-
-pg8000.dbapi.Cursor.description
-'''''''''''''''''''''''''''''''
-
-This read-only attribute is a sequence of 7-item sequences. Each value contains
-information describing one result column. The 7 items returned for each column are
-(name, type_code, display_size, internal_size, precision, scale, null_ok). Only the
-first two values are provided by the current implementation.
-
-
-pg8000.dbapi.Cursor.close()
-'''''''''''''''''''''''''''
-
-Closes the cursor.
-
-
-pg8000.dbapi.Cursor.execute(operation, args=None, stream=None)
-''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Executes a database operation. Parameters may be provided as a sequence, or as a
-mapping, depending upon the value of ``pg8000.dbapi.paramstyle``. Returns the cursor,
-which may be iterated over.
-
-operation
- The SQL statement to execute.
-
-args
- If ``pg8000.dbapi.paramstyle`` is ``qmark``, ``numeric``, or ``format``, this
- argument should be an array of parameters to bind into the statement. If
- ``pg8000.dbapi.paramstyle`` is ``named``, the argument should be a ``dict`` mapping of
- parameters. If ``pg8000.dbapi.paramstyle`` is ``pyformat``, the argument value may be
- either an array or a mapping.
-
-stream
- This is a pg8000 extension for use with the PostgreSQL `COPY
- <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. For a
- ``COPY FROM`` the parameter must be a readable file-like object, and for ``COPY TO``
- it must be writable.
-
-
-pg8000.dbapi.Cursor.executemany(operation, param_sets)
-''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Prepare a database operation, and then execute it against all parameter sequences or
-mappings provided.
-
-operation
- The SQL statement to execute.
-
-parameter_sets
- A sequence of parameters to execute the statement with. The values in the sequence
- should be sequences or mappings of parameters, the same as the args argument of the
- ``pg8000.dbapi.Cursor.execute()`` method.
-
-
-pg8000.dbapi.Cursor.callproc(procname, parameters=None)
-'''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Call a stored database procedure with the given name and optional parameters.
-
-
-procname
- The name of the procedure to call.
-
-parameters
- A list of parameters.
-
-
-pg8000.dbapi.Cursor.fetchall()
-''''''''''''''''''''''''''''''
-
-Fetches all remaining rows of a query result.
-
-Returns: A sequence, each entry of which is a sequence of field values making up a row.
-
-
-pg8000.dbapi.Cursor.fetchmany(size=None)
-''''''''''''''''''''''''''''''''''''''''
-
-Fetches the next set of rows of a query result.
-
-size
- The number of rows to fetch when called. If not provided, the
- ``pg8000.dbapi.Cursor.arraysize`` attribute value is used instead.
-
-Returns: A sequence, each entry of which is a sequence of field values making up a row.
-If no more rows are available, an empty sequence will be returned.
-
-
-pg8000.dbapi.Cursor.fetchone()
-''''''''''''''''''''''''''''''
-
-Fetch the next row of a query result set.
-
-Returns: A row as a sequence of field values, or ``None`` if no more rows are available.
-
-
-pg8000.dbapi.Cursor.setinputsizes(\*sizes)
-''''''''''''''''''''''''''''''''''''''''''
-
-Used to set the parameter types of the next query. This is useful if it's difficult for
-pg8000 to work out the types from the parameters themselves (eg. for parameters of type
-None).
-
-sizes
- Positional parameters that are either the Python type of the parameter to be sent, or
- the PostgreSQL oid. Common oids are available as constants such as ``pg8000.STRING``,
- ``pg8000.INTEGER``, ``pg8000.TIME`` etc.
-
-
-pg8000.dbapi.Cursor.setoutputsize(size, column=None)
-''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Not implemented by pg8000.
-
-
-pg8000.dbapi.Interval
-'''''''''''''''''''''
-
-An Interval represents a measurement of time. In PostgreSQL, an interval is defined in
-the measure of months, days, and microseconds; as such, the pg8000 interval type
-represents the same information.
-
-Note that values of the ``pg8000.dbapi.Interval.microseconds``,
-``pg8000.dbapi.Interval.days``, and ``pg8000.dbapi.Interval.months`` properties are
-independently measured and cannot be converted to each other. A month may be 28, 29, 30,
-or 31 days, and a day may occasionally be lengthened slightly by a leap second.
+`DB-API 2 Docs <docs/dbapi2_docs.rst>`_
Design Decisions
@@ -2258,243 +1432,4 @@ Run ``tox`` to make sure all tests pass, then update the release notes, then do:
Release Notes
-------------
-Version 1.30.4, 2024-01-03
-``````````````````````````
-
-- Add support for more range and multirange types.
-
-- Make the ``Connection.parameter_statuses`` property a ``dict`` rather than a ``dequeue``.
-
-
-Version 1.30.3, 2023-10-31
-``````````````````````````
-
-- Fix problem with PG date overflowing Python types. Now we return the ``str`` we got from the
- server if we can't parse it.
-
-
-Version 1.30.2, 2023-09-17
-``````````````````````````
-
-- Bug fix where dollar-quoted string constants weren't supported.
-
-
-Version 1.30.1, 2023-07-29
-``````````````````````````
-
-- There was a problem uploading the previous version (1.30.0) to PyPI because the
- markup of the README.rst was invalid. There's now a step in the automated tests to
- check for this.
-
-
-Version 1.30.0, 2023-07-27
-``````````````````````````
-
-- Remove support for Python 3.7
-
-- Add a ``sock`` keyword parameter for creating a connection from a pre-configured
- socket.
-
-
-Version 1.29.8, 2023-06-16
-``````````````````````````
-
-- Ranges don't work with legacy API.
-
-
-Version 1.29.7, 2023-06-16
-``````````````````````````
-
-- Add support for PostgreSQL ``range`` and ``multirange`` types. Previously pg8000
- would just return them as strings, but now they're returned as ``Range`` and lists of
- ``Range``.
-
-- The PostgreSQL ``record`` type is now returned as a ``tuple`` of strings, whereas
- before it was returned as one string.
-
-
-Version 1.29.6, 2023-05-29
-``````````````````````````
-
-- Fixed two bugs with composite types. Nulls should be represented by an empty string,
- and in an array of composite types, the elements should be surrounded by double
- quotes.
-
-
-Version 1.29.5, 2023-05-09
-``````````````````````````
-
-- Fixed bug where pg8000 didn't handle the case when the number of bytes received from
- a socket was fewer than requested. This was being interpreted as a network error, but
- in fact we just needed to wait until more bytes were available.
-
-- When using the ``PGInterval`` type, if a response from the server contained the period
- ``millennium``, it wasn't recognised. This was caused by a spelling mistake where we
- had ``millenium`` rather than ``millennium``.
-
-- Added support for sending PostgreSQL composite types. If a value is sent as a
- ``tuple``, pg8000 will send it to the server as a ``(`` delimited composite string.
-
-
-Version 1.29.4, 2022-12-14
-``````````````````````````
-
-- Fixed bug in ``pg8000.dbapi`` in the ``setinputsizes()`` method where if a ``size``
- was a recognized Python type, the method failed.
-
-
-Version 1.29.3, 2022-10-26
-``````````````````````````
-
-- Upgrade the SCRAM library to version 1.4.3. This adds support for the case where the
- client supports channel binding but the server doesn't.
-
-
-Version 1.29.2, 2022-10-09
-``````````````````````````
-
-- Fixed a bug where in a literal array, items such as ``\n`` and ``\r`` weren't
- escaped properly before being sent to the server.
-
-- Fixed a bug where if the PostgreSQL server has a half-hour time zone set, values of
- type ``timestamp with time zone`` failed. This has been fixed by using the ``parse``
- function of the ``dateutil`` package if the ``datetime`` parser fails.
-
-
-Version 1.29.1, 2022-05-23
-``````````````````````````
-
-- In trying to determine if there's been a failed commit, check for ``ROLLBACK TO
- SAVEPOINT``.
-
-
-Version 1.29.0, 2022-05-21
-``````````````````````````
-
-- Implement a workaround for the `silent failed commit
- <https://github.com/tlocke/pg8000/issues/36>`_ bug.
-
-- Previously if an empty string was sent as the query an exception would be raised, but
- that isn't done now.
-
-
-Version 1.28.3, 2022-05-18
-``````````````````````````
-
-- Put back ``__version__`` attributes that were inadvertently removed.
-
-
-Version 1.28.2, 2022-05-17
-``````````````````````````
-
-- Use a build system that's compliant with PEP517.
-
-
-Version 1.28.1, 2022-05-17
-``````````````````````````
-
-- If when doing a ``COPY FROM`` the ``stream`` parameter is an iterator of ``str``,
- pg8000 used to silently append a newline to the end. That no longer happens.
-
-
-Version 1.28.0, 2022-05-17
-``````````````````````````
-
-- When using the ``COPY FROM`` SQL statement, allow the ``stream`` parameter to be an
- iterable.
-
-
-Version 1.27.1, 2022-05-16
-``````````````````````````
-
-- The ``seconds`` attribute of ``PGInterval`` is now always a ``float``, to cope with
- fractional seconds.
-
-- Updated the ``interval`` parsers for ``iso_8601`` and ``sql_standard`` to take
- account of fractional seconds.
-
-
-Version 1.27.0, 2022-05-16
-``````````````````````````
-
-- It used to be that by default, if pg8000 received an ``interval`` type from the server
- and it was too big to fit into a ``datetime.timedelta`` then an exception would be
- raised. Now if an interval is too big for ``datetime.timedelta`` a ``PGInterval`` is
- returned.
-
-* pg8000 now supports all the output formats for an ``interval`` (``postgres``,
- ``postgres_verbose``, ``iso_8601`` and ``sql_standard``).
-
-
-Version 1.26.1, 2022-04-23
-``````````````````````````
-
-- Make sure all tests are run by the GitHub Actions tests on commit.
-- Remove support for Python 3.6
-- Remove support for PostgreSQL 9.6
-
-
-Version 1.26.0, 2022-04-18
-``````````````````````````
-
-- When connecting, raise an ``InterfaceError('network error')`` rather than let the
- underlying ``struct.error`` float up.
-
-- Make licence text the same as that used by the OSI. Previously the licence wording
- differed slightly from the BSD 3 Clause licence at
- https://opensource.org/licenses/BSD-3-Clause. This meant that automated tools didn't
- pick it up as being Open Source. The changes are believed to not alter the meaning of the license at all.
-
-
-Version 1.25.0, 2022-04-17
-``````````````````````````
-
-- Fix more cases where a ``ResourceWarning`` would be raise because of a socket that had
- been left open.
-
-- We now have a single ``InterfaceError`` with the message 'network error' for all
- network errors, with the underlying exception held in the ``cause`` of the exception.
-
-
-Version 1.24.2, 2022-04-15
-``````````````````````````
-
-- To prevent a ``ResourceWarning`` close socket if a connection can't be created.
-
-
-Version 1.24.1, 2022-03-02
-``````````````````````````
-
-- Return pg +/-infinity dates as ``str``. Previously +/-infinity pg values would cause
- an error when returned, but now we return +/-infinity as strings.
-
-
-Version 1.24.0, 2022-02-06
-``````````````````````````
-
-- Add SQL escape functions identifier() and literal() to the native API. For use when a
- query can't be parameterised and the SQL string has to be created using untrusted
- values.
-
-
-Version 1.23.0, 2021-11-13
-``````````````````````````
-
-- If a query has no parameters, then the query will no longer be parsed. Although there
- are performance benefits for doing this, the main reason is to avoid query rewriting,
- which can introduce errors.
-
-
-Version 1.22.1, 2021-11-10
-``````````````````````````
-
-- Fix bug in PGInterval type where ``str()`` failed for a millennia value.
-
-
-Version 1.22.0, 2021-10-13
-``````````````````````````
-
-- Rather than specifying the oids in the ``Parse`` step of the Postgres protocol, pg8000
- now omits them, and so Postgres will use the oids it determines from the query. This
- makes the pg8000 code simpler and also it should also make the nuances of type
- matching more straightforward.
+`Release Notes <docs/release_notes.rst>`_
diff --git a/contrib/python/pg8000/README.rst b/contrib/python/pg8000/README.rst
index ad8ea650f2..55c26ddef3 100644
--- a/contrib/python/pg8000/README.rst
+++ b/contrib/python/pg8000/README.rst
@@ -1323,839 +1323,13 @@ pg8000 encounters a network problem it'll raise an ``InterfaceError`` with the m
Native API Docs
---------------
-pg8000.native.Error
-```````````````````
-
-Generic exception that is the base exception of the other error exceptions.
-
-
-pg8000.native.InterfaceError
-````````````````````````````
-
-For errors that originate within pg8000.
-
-
-pg8000.native.DatabaseError
-```````````````````````````
-
-For errors that originate from the server.
-
-pg8000.native.Connection(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None)
-``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````
-
-Creates a connection to a PostgreSQL database.
-
-user
- The username to connect to the PostgreSQL server with. If your server character
- encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes,
- eg. ``'my_name'.encode('EUC-JP')``.
-
-host
- The hostname of the PostgreSQL server to connect with. Providing this parameter is
- necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be
- provided. The default is ``localhost``.
-
-database
- The name of the database instance to connect with. If ``None`` then the PostgreSQL
- server will assume the database name is the same as the username. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database``
- as bytes, eg. ``'my_db'.encode('EUC-JP')``.
-
-port
- The TCP/IP port of the PostgreSQL server instance. This parameter defaults to
- ``5432``, the registered common port of PostgreSQL TCP/IP servers.
-
-password
- The user password to connect to the server with. This parameter is optional; if
- omitted and the database server requests password-based authentication, the connection
- will fail to open. If this parameter is provided but not
- requested by the server, no error will occur.
-
- If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``.
-
-source_address
- The source IP address which initiates the connection to the PostgreSQL server. The
- default is ``None`` which means that the operating system will choose the source
- address.
-
-unix_sock
- The path to the UNIX socket to access the database through, for example,
- ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided.
-
-ssl_context
- This governs SSL encryption for TCP/IP sockets. It can have three values:
-
- - ``None``, meaning no SSL (the default)
-
- - ``True``, means use SSL with an |ssl.SSLContext|_ created using
- |ssl.create_default_context()|_
-
- - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection.
-
- If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific
- attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL
- socket, but not to request SSL from the PostgreSQL server. Note that this means you
- can't use SCRAM authentication with channel binding.
-
-timeout
- This is the time in seconds before the connection to the server will time out. The
- default is ``None`` which means no timeout.
-
-tcp_keepalive
- If ``True`` then use `TCP keepalive
- <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``.
-
-application_name
- Sets the `application_name
- <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_.
- If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default
- is ``None`` which means that the server will set the application name.
-
-replication
- Used to run in `streaming replication mode
- <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide values as
- bytes, eg. ``'database'.encode('EUC-JP')``.
-
-sock
- A socket-like object to use for the connection. For example, ``sock`` could be a plain
- ``socket.socket``, or it could represent an SSH tunnel or perhaps an
- ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be
- used to attempt to create an SSL socket from the provided socket.
-
-pg8000.native.Connection.notifications
-``````````````````````````````````````
-
-A deque of server-side `notifications
-<https://www.postgresql.org/docs/current/sql-notify.html>`__ received by this database
-connection (via the ``LISTEN`` / ``NOTIFY`` PostgreSQL commands). Each list item is a
-three-element tuple containing the PostgreSQL backend PID that issued the notify, the
-channel and the payload.
-
-
-pg8000.native.Connection.notices
-````````````````````````````````
-
-A deque of server-side notices received by this database connection.
-
-
-pg8000.native.Connection.parameter_statuses
-```````````````````````````````````````````
-
-A deque of server-side parameter statuses received by this database connection.
-
-
-pg8000.native.Connection.run(sql, stream=None, types=None, \*\*kwargs)
-``````````````````````````````````````````````````````````````````````
-
-Executes an sql statement, and returns the results as a ``list``. For example::
-
- con.run("SELECT * FROM cities where population > :pop", pop=10000)
-
-sql
- The SQL statement to execute. Parameter placeholders appear as a ``:`` followed by the
- parameter name.
-
-stream
- For use with the PostgreSQL `COPY
- <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. The nature
- of the parameter depends on whether the SQL command is ``COPY FROM`` or ``COPY TO``.
-
- ``COPY FROM``
- The stream parameter must be a readable file-like object or an iterable. If it's an
- iterable then the items can be ``str`` or binary.
- ``COPY TO``
- The stream parameter must be a writable file-like object.
-
-types
- A dictionary of oids. A key corresponds to a parameter.
-
-kwargs
- The parameters of the SQL statement.
-
-
-pg8000.native.Connection.row_count
-``````````````````````````````````
-
-This read-only attribute contains the number of rows that the last ``run()`` method
-produced (for query statements like ``SELECT``) or affected (for modification statements
-like ``UPDATE``.
-
-The value is -1 if:
-
-- No ``run()`` method has been performed yet.
-- There was no rowcount associated with the last ``run()``.
-
-
-pg8000.native.Connection.columns
-````````````````````````````````
-
-A list of column metadata. Each item in the list is a dictionary with the following
-keys:
-
-- name
-- table_oid
-- column_attrnum
-- type_oid
-- type_size
-- type_modifier
-- format
-
-
-pg8000.native.Connection.close()
-````````````````````````````````
-
-Closes the database connection.
-
-
-pg8000.native.Connection.register_out_adapter(typ, out_func)
-````````````````````````````````````````````````````````````
-
-Register a type adapter for types going out from pg8000 to the server.
-
-typ
- The Python class that the adapter is for.
-
-out_func
- A function that takes the Python object and returns its string representation
- in the format that the server requires.
-
-
-pg8000.native.Connection.register_in_adapter(oid, in_func)
-``````````````````````````````````````````````````````````
-
-Register a type adapter for types coming in from the server to pg8000.
-
-oid
- The PostgreSQL type identifier found in the `pg_type system catalog
- <https://www.postgresql.org/docs/current/catalog-pg-type.html>`_.
-
-in_func
- A function that takes the PostgreSQL string representation and returns a corresponding
- Python object.
-
-
-pg8000.native.Connection.prepare(sql)
-`````````````````````````````````````
-
-Returns a ``PreparedStatement`` object which represents a `prepared statement
-<https://www.postgresql.org/docs/current/sql-prepare.html>`_ on the server. It can
-subsequently be repeatedly executed.
-
-sql
- The SQL statement to prepare. Parameter placeholders appear as a ``:`` followed by the
- parameter name.
-
-
-pg8000.native.PreparedStatement
-```````````````````````````````
-
-A prepared statement object is returned by the ``pg8000.native.Connection.prepare()``
-method of a connection. It has the following methods:
-
-
-pg8000.native.PreparedStatement.run(\*\*kwargs)
-```````````````````````````````````````````````
-
-Executes the prepared statement, and returns the results as a ``tuple``.
-
-kwargs
- The parameters of the prepared statement.
-
-
-pg8000.native.PreparedStatement.close()
-```````````````````````````````````````
-
-Closes the prepared statement, releasing the prepared statement held on the server.
-
-
-pg8000.native.identifier(ident)
-```````````````````````````````
-
-Correctly quotes and escapes a string to be used as an `SQL identifier
-<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS>`_.
-
-ident
- The ``str`` to be used as an SQL identifier.
-
-
-pg8000.native.literal(value)
-````````````````````````````
-
-Correctly quotes and escapes a value to be used as an `SQL literal
-<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS>`_.
-
-value
- The value to be used as an SQL literal.
+`Native API Docs <docs/native_api_docs.rst>`_
DB-API 2 Docs
-------------
-
-Properties
-``````````
-
-
-pg8000.dbapi.apilevel
-:::::::::::::::::::::
-
-The DBAPI level supported, currently "2.0".
-
-
-pg8000.dbapi.threadsafety
-:::::::::::::::::::::::::
-
-Integer constant stating the level of thread safety the DBAPI interface supports. For
-pg8000, the threadsafety value is 1, meaning that threads may share the module but not
-connections.
-
-
-pg8000.dbapi.paramstyle
-:::::::::::::::::::::::
-
-String property stating the type of parameter marker formatting expected by
-the interface. This value defaults to "format", in which parameters are
-marked in this format: "WHERE name=%s".
-
-As an extension to the DBAPI specification, this value is not constant; it can be
-changed to any of the following values:
-
-qmark
- Question mark style, eg. ``WHERE name=?``
-
-numeric
- Numeric positional style, eg. ``WHERE name=:1``
-
-named
- Named style, eg. ``WHERE name=:paramname``
-
-format
- printf format codes, eg. ``WHERE name=%s``
-
-pyformat
- Python format codes, eg. ``WHERE name=%(paramname)s``
-
-
-pg8000.dbapi.STRING
-:::::::::::::::::::
-
-String type oid.
-
-pg8000.dbapi.BINARY
-:::::::::::::::::::
-
-
-pg8000.dbapi.NUMBER
-:::::::::::::::::::
-
-Numeric type oid.
-
-
-pg8000.dbapi.DATETIME
-:::::::::::::::::::::
-
-Timestamp type oid
-
-
-pg8000.dbapi.ROWID
-::::::::::::::::::
-
-ROWID type oid
-
-
-Functions
-`````````
-
-pg8000.dbapi.connect(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None)
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-Creates a connection to a PostgreSQL database.
-
-user
- The username to connect to the PostgreSQL server with. If your server character
- encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes,
- eg. ``'my_name'.encode('EUC-JP')``.
-
-host
- The hostname of the PostgreSQL server to connect with. Providing this parameter is
- necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be
- provided. The default is ``localhost``.
-
-database
- The name of the database instance to connect with. If ``None`` then the PostgreSQL
- server will assume the database name is the same as the username. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database``
- as bytes, eg. ``'my_db'.encode('EUC-JP')``.
-
-port
- The TCP/IP port of the PostgreSQL server instance. This parameter defaults to
- ``5432``, the registered common port of PostgreSQL TCP/IP servers.
-
-password
- The user password to connect to the server with. This parameter is optional; if
- omitted and the database server requests password-based authentication, the
- connection will fail to open. If this parameter is provided but not requested by the
- server, no error will occur.
-
- If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``.
-
-source_address
- The source IP address which initiates the connection to the PostgreSQL server. The
- default is ``None`` which means that the operating system will choose the source
- address.
-
-unix_sock
- The path to the UNIX socket to access the database through, for example,
- ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided.
-
-ssl_context
- This governs SSL encryption for TCP/IP sockets. It can have three values:
-
- - ``None``, meaning no SSL (the default)
- - ``True``, means use SSL with an |ssl.SSLContext|_ created using
- |ssl.create_default_context()|_.
-
- - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection.
-
- If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific
- attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL
- socket, but not to request SSL from the PostgreSQL server. Note that this means you
- can't use SCRAM authentication with channel binding.
-
-timeout
- This is the time in seconds before the connection to the server will time out. The
- default is ``None`` which means no timeout.
-
-tcp_keepalive
- If ``True`` then use `TCP keepalive
- <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``.
-
-application_name
- Sets the `application_name
- <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_. If your server character encoding is not ``ascii`` or ``utf8``, then you need to
- provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default
- is ``None`` which means that the server will set the application name.
-
-replication
- Used to run in `streaming replication mode
- <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server
- character encoding is not ``ascii`` or ``utf8``, then you need to provide values as
- bytes, eg. ``'database'.encode('EUC-JP')``.
-
-sock
- A socket-like object to use for the connection. For example, ``sock`` could be a plain
- ``socket.socket``, or it could represent an SSH tunnel or perhaps an
- ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be
- used to attempt to create an SSL socket from the provided socket.
-
-
-pg8000.dbapi.Date(year, month, day)
-
-Construct an object holding a date value.
-
-This property is part of the `DBAPI 2.0 specification
-<http://www.python.org/dev/peps/pep-0249/>`_.
-
-Returns: `datetime.date`
-
-
-pg8000.dbapi.Time(hour, minute, second)
-:::::::::::::::::::::::::::::::::::::::
-
-Construct an object holding a time value.
-
-Returns: ``datetime.time``
-
-
-pg8000.dbapi.Timestamp(year, month, day, hour, minute, second)
-::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-Construct an object holding a timestamp value.
-
-Returns: ``datetime.datetime``
-
-
-pg8000.dbapi.DateFromTicks(ticks)
-:::::::::::::::::::::::::::::::::
-
-Construct an object holding a date value from the given ticks value (number of seconds
-since the epoch).
-
-Returns: ``datetime.datetime``
-
-
-pg8000.dbapi.TimeFromTicks(ticks)
-:::::::::::::::::::::::::::::::::
-
-Construct an object holding a time value from the given ticks value (number of seconds
-since the epoch).
-
-Returns: ``datetime.time``
-
-
-pg8000.dbapi.TimestampFromTicks(ticks)
-::::::::::::::::::::::::::::::::::::::
-
-Construct an object holding a timestamp value from the given ticks value (number of
-seconds since the epoch).
-
-Returns: ``datetime.datetime``
-
-
-pg8000.dbapi.Binary(value)
-::::::::::::::::::::::::::
-
-Construct an object holding binary data.
-
-Returns: ``bytes``.
-
-
-Generic Exceptions
-``````````````````
-
-Pg8000 uses the standard DBAPI 2.0 exception tree as "generic" exceptions. Generally,
-more specific exception types are raised; these specific exception types are derived
-from the generic exceptions.
-
-pg8000.dbapi.Warning
-::::::::::::::::::::
-
-Generic exception raised for important database warnings like data truncations. This
-exception is not currently used by pg8000.
-
-
-pg8000.dbapi.Error
-::::::::::::::::::
-
-Generic exception that is the base exception of all other error exceptions.
-
-
-pg8000.dbapi.InterfaceError
-:::::::::::::::::::::::::::
-
-Generic exception raised for errors that are related to the database interface rather
-than the database itself. For example, if the interface attempts to use an SSL
-connection but the server refuses, an InterfaceError will be raised.
-
-
-pg8000.dbapi.DatabaseError
-::::::::::::::::::::::::::
-
-Generic exception raised for errors that are related to the database. This exception is
-currently never raised by pg8000.
-
-
-pg8000.dbapi.DataError
-::::::::::::::::::::::
-
-Generic exception raised for errors that are due to problems with the processed data.
-This exception is not currently raised by pg8000.
-
-
-pg8000.dbapi.OperationalError
-:::::::::::::::::::::::::::::
-
-Generic exception raised for errors that are related to the database's operation and not
-necessarily under the control of the programmer. This exception is currently never
-raised by pg8000.
-
-
-pg8000.dbapi.IntegrityError
-:::::::::::::::::::::::::::
-
-Generic exception raised when the relational integrity of the database is affected. This
-exception is not currently raised by pg8000.
-
-
-pg8000.dbapi.InternalError
-::::::::::::::::::::::::::
-
-Generic exception raised when the database encounters an internal error. This is
-currently only raised when unexpected state occurs in the pg8000 interface itself, and
-is typically the result of a interface bug.
-
-
-pg8000.dbapi.ProgrammingError
-:::::::::::::::::::::::::::::
-
-Generic exception raised for programming errors. For example, this exception is raised
-if more parameter fields are in a query string than there are available parameters.
-
-
-pg8000.dbapi.NotSupportedError
-::::::::::::::::::::::::::::::
-
-Generic exception raised in case a method or database API was used which is not
-supported by the database.
-
-
-Classes
-```````
-
-
-pg8000.dbapi.Connection
-:::::::::::::::::::::::
-
-A connection object is returned by the ``pg8000.connect()`` function. It represents a
-single physical connection to a PostgreSQL database.
-
-
-pg8000.dbapi.Connection.autocommit
-::::::::::::::::::::::::::::::::::
-
-Following the DB-API specification, autocommit is off by default. It can be turned on by
-setting this boolean pg8000-specific autocommit property to ``True``.
-
-
-pg8000.dbapi.Connection.close()
-:::::::::::::::::::::::::::::::
-
-Closes the database connection.
-
-
-pg8000.dbapi.Connection.cursor()
-::::::::::::::::::::::::::::::::
-
-Creates a ``pg8000.dbapi.Cursor`` object bound to this connection.
-
-
-pg8000.dbapi.Connection.rollback()
-::::::::::::::::::::::::::::::::::
-
-Rolls back the current database transaction.
-
-
-pg8000.dbapi.Connection.tpc_begin(xid)
-::::::::::::::::::::::::::::::::::::::
-
-Begins a TPC transaction with the given transaction ID xid. This method should be
-called outside of a transaction (i.e. nothing may have executed since the last
-``commit()`` or ``rollback()``. Furthermore, it is an error to call ``commit()`` or
-``rollback()`` within the TPC transaction. A ``ProgrammingError`` is raised, if the
-application calls ``commit()`` or ``rollback()`` during an active TPC transaction.
-
-
-pg8000.dbapi.Connection.tpc_commit(xid=None)
-::::::::::::::::::::::::::::::::::::::::::::
-
-When called with no arguments, ``tpc_commit()`` commits a TPC transaction previously
-prepared with ``tpc_prepare()``. If ``tpc_commit()`` is called prior to
-``tpc_prepare()``, a single phase commit is performed. A transaction manager may choose
-to do this if only a single resource is participating in the global transaction.
-
-When called with a transaction ID ``xid``, the database commits the given transaction.
-If an invalid transaction ID is provided, a ``ProgrammingError`` will be raised. This
-form should be called outside of a transaction, and is intended for use in recovery.
-
-On return, the TPC transaction is ended.
-
-
-pg8000.dbapi.Connection.tpc_prepare()
-:::::::::::::::::::::::::::::::::::::
-
-Performs the first phase of a transaction started with ``.tpc_begin()``. A
-``ProgrammingError`` is be raised if this method is called outside of a TPC transaction.
-
-After calling ``tpc_prepare()``, no statements can be executed until ``tpc_commit()`` or
-``tpc_rollback()`` have been called.
-
-
-pg8000.dbapi.Connection.tpc_recover()
-:::::::::::::::::::::::::::::::::::::
-
-Returns a list of pending transaction IDs suitable for use with ``tpc_commit(xid)`` or
-``tpc_rollback(xid)``.
-
-
-pg8000.dbapi.Connection.tpc_rollback(xid=None)
-::::::::::::::::::::::::::::::::::::::::::::::
-
-When called with no arguments, ``tpc_rollback()`` rolls back a TPC transaction. It may
-be called before or after ``tpc_prepare()``.
-
-When called with a transaction ID xid, it rolls back the given transaction. If an
-invalid transaction ID is provided, a ``ProgrammingError`` is raised. This form should
-be called outside of a transaction, and is intended for use in recovery.
-
-On return, the TPC transaction is ended.
-
-
-pg8000.dbapi.Connection.xid(format_id, global_transaction_id, branch_qualifier)
-:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-Create a Transaction IDs (only global_transaction_id is used in pg) format_id and
-branch_qualifier are not used in postgres global_transaction_id may be any string
-identifier supported by postgres returns a tuple (format_id, global_transaction_id,
-branch_qualifier)
-
-
-pg8000.dbapi.Cursor
-:::::::::::::::::::
-
-A cursor object is returned by the ``pg8000.dbapi.Connection.cursor()`` method of a
-connection. It has the following attributes and methods:
-
-pg8000.dbapi.Cursor.arraysize
-'''''''''''''''''''''''''''''
-
-This read/write attribute specifies the number of rows to fetch at a time with
-``pg8000.dbapi.Cursor.fetchmany()``. It defaults to 1.
-
-
-pg8000.dbapi.Cursor.connection
-''''''''''''''''''''''''''''''
-
-This read-only attribute contains a reference to the connection object (an instance of
-``pg8000.dbapi.Connection``) on which the cursor was created.
-
-
-pg8000.dbapi.Cursor.rowcount
-''''''''''''''''''''''''''''
-
-This read-only attribute contains the number of rows that the last ``execute()`` or
-``executemany()`` method produced (for query statements like ``SELECT``) or affected
-(for modification statements like ``UPDATE``.
-
-The value is -1 if:
-
-- No ``execute()`` or ``executemany()`` method has been performed yet on the cursor.
-
-- There was no rowcount associated with the last ``execute()``.
-
-- At least one of the statements executed as part of an ``executemany()`` had no row
- count associated with it.
-
-
-pg8000.dbapi.Cursor.description
-'''''''''''''''''''''''''''''''
-
-This read-only attribute is a sequence of 7-item sequences. Each value contains
-information describing one result column. The 7 items returned for each column are
-(name, type_code, display_size, internal_size, precision, scale, null_ok). Only the
-first two values are provided by the current implementation.
-
-
-pg8000.dbapi.Cursor.close()
-'''''''''''''''''''''''''''
-
-Closes the cursor.
-
-
-pg8000.dbapi.Cursor.execute(operation, args=None, stream=None)
-''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Executes a database operation. Parameters may be provided as a sequence, or as a
-mapping, depending upon the value of ``pg8000.dbapi.paramstyle``. Returns the cursor,
-which may be iterated over.
-
-operation
- The SQL statement to execute.
-
-args
- If ``pg8000.dbapi.paramstyle`` is ``qmark``, ``numeric``, or ``format``, this
- argument should be an array of parameters to bind into the statement. If
- ``pg8000.dbapi.paramstyle`` is ``named``, the argument should be a ``dict`` mapping of
- parameters. If ``pg8000.dbapi.paramstyle`` is ``pyformat``, the argument value may be
- either an array or a mapping.
-
-stream
- This is a pg8000 extension for use with the PostgreSQL `COPY
- <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. For a
- ``COPY FROM`` the parameter must be a readable file-like object, and for ``COPY TO``
- it must be writable.
-
-
-pg8000.dbapi.Cursor.executemany(operation, param_sets)
-''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Prepare a database operation, and then execute it against all parameter sequences or
-mappings provided.
-
-operation
- The SQL statement to execute.
-
-parameter_sets
- A sequence of parameters to execute the statement with. The values in the sequence
- should be sequences or mappings of parameters, the same as the args argument of the
- ``pg8000.dbapi.Cursor.execute()`` method.
-
-
-pg8000.dbapi.Cursor.callproc(procname, parameters=None)
-'''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Call a stored database procedure with the given name and optional parameters.
-
-
-procname
- The name of the procedure to call.
-
-parameters
- A list of parameters.
-
-
-pg8000.dbapi.Cursor.fetchall()
-''''''''''''''''''''''''''''''
-
-Fetches all remaining rows of a query result.
-
-Returns: A sequence, each entry of which is a sequence of field values making up a row.
-
-
-pg8000.dbapi.Cursor.fetchmany(size=None)
-''''''''''''''''''''''''''''''''''''''''
-
-Fetches the next set of rows of a query result.
-
-size
- The number of rows to fetch when called. If not provided, the
- ``pg8000.dbapi.Cursor.arraysize`` attribute value is used instead.
-
-Returns: A sequence, each entry of which is a sequence of field values making up a row.
-If no more rows are available, an empty sequence will be returned.
-
-
-pg8000.dbapi.Cursor.fetchone()
-''''''''''''''''''''''''''''''
-
-Fetch the next row of a query result set.
-
-Returns: A row as a sequence of field values, or ``None`` if no more rows are available.
-
-
-pg8000.dbapi.Cursor.setinputsizes(\*sizes)
-''''''''''''''''''''''''''''''''''''''''''
-
-Used to set the parameter types of the next query. This is useful if it's difficult for
-pg8000 to work out the types from the parameters themselves (eg. for parameters of type
-None).
-
-sizes
- Positional parameters that are either the Python type of the parameter to be sent, or
- the PostgreSQL oid. Common oids are available as constants such as ``pg8000.STRING``,
- ``pg8000.INTEGER``, ``pg8000.TIME`` etc.
-
-
-pg8000.dbapi.Cursor.setoutputsize(size, column=None)
-''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-Not implemented by pg8000.
-
-
-pg8000.dbapi.Interval
-'''''''''''''''''''''
-
-An Interval represents a measurement of time. In PostgreSQL, an interval is defined in
-the measure of months, days, and microseconds; as such, the pg8000 interval type
-represents the same information.
-
-Note that values of the ``pg8000.dbapi.Interval.microseconds``,
-``pg8000.dbapi.Interval.days``, and ``pg8000.dbapi.Interval.months`` properties are
-independently measured and cannot be converted to each other. A month may be 28, 29, 30,
-or 31 days, and a day may occasionally be lengthened slightly by a leap second.
+`DB-API 2 Docs <docs/dbapi2_docs.rst>`_
Design Decisions
@@ -2230,243 +1404,4 @@ Run ``tox`` to make sure all tests pass, then update the release notes, then do:
Release Notes
-------------
-Version 1.30.4, 2024-01-03
-``````````````````````````
-
-- Add support for more range and multirange types.
-
-- Make the ``Connection.parameter_statuses`` property a ``dict`` rather than a ``dequeue``.
-
-
-Version 1.30.3, 2023-10-31
-``````````````````````````
-
-- Fix problem with PG date overflowing Python types. Now we return the ``str`` we got from the
- server if we can't parse it.
-
-
-Version 1.30.2, 2023-09-17
-``````````````````````````
-
-- Bug fix where dollar-quoted string constants weren't supported.
-
-
-Version 1.30.1, 2023-07-29
-``````````````````````````
-
-- There was a problem uploading the previous version (1.30.0) to PyPI because the
- markup of the README.rst was invalid. There's now a step in the automated tests to
- check for this.
-
-
-Version 1.30.0, 2023-07-27
-``````````````````````````
-
-- Remove support for Python 3.7
-
-- Add a ``sock`` keyword parameter for creating a connection from a pre-configured
- socket.
-
-
-Version 1.29.8, 2023-06-16
-``````````````````````````
-
-- Ranges don't work with legacy API.
-
-
-Version 1.29.7, 2023-06-16
-``````````````````````````
-
-- Add support for PostgreSQL ``range`` and ``multirange`` types. Previously pg8000
- would just return them as strings, but now they're returned as ``Range`` and lists of
- ``Range``.
-
-- The PostgreSQL ``record`` type is now returned as a ``tuple`` of strings, whereas
- before it was returned as one string.
-
-
-Version 1.29.6, 2023-05-29
-``````````````````````````
-
-- Fixed two bugs with composite types. Nulls should be represented by an empty string,
- and in an array of composite types, the elements should be surrounded by double
- quotes.
-
-
-Version 1.29.5, 2023-05-09
-``````````````````````````
-
-- Fixed bug where pg8000 didn't handle the case when the number of bytes received from
- a socket was fewer than requested. This was being interpreted as a network error, but
- in fact we just needed to wait until more bytes were available.
-
-- When using the ``PGInterval`` type, if a response from the server contained the period
- ``millennium``, it wasn't recognised. This was caused by a spelling mistake where we
- had ``millenium`` rather than ``millennium``.
-
-- Added support for sending PostgreSQL composite types. If a value is sent as a
- ``tuple``, pg8000 will send it to the server as a ``(`` delimited composite string.
-
-
-Version 1.29.4, 2022-12-14
-``````````````````````````
-
-- Fixed bug in ``pg8000.dbapi`` in the ``setinputsizes()`` method where if a ``size``
- was a recognized Python type, the method failed.
-
-
-Version 1.29.3, 2022-10-26
-``````````````````````````
-
-- Upgrade the SCRAM library to version 1.4.3. This adds support for the case where the
- client supports channel binding but the server doesn't.
-
-
-Version 1.29.2, 2022-10-09
-``````````````````````````
-
-- Fixed a bug where in a literal array, items such as ``\n`` and ``\r`` weren't
- escaped properly before being sent to the server.
-
-- Fixed a bug where if the PostgreSQL server has a half-hour time zone set, values of
- type ``timestamp with time zone`` failed. This has been fixed by using the ``parse``
- function of the ``dateutil`` package if the ``datetime`` parser fails.
-
-
-Version 1.29.1, 2022-05-23
-``````````````````````````
-
-- In trying to determine if there's been a failed commit, check for ``ROLLBACK TO
- SAVEPOINT``.
-
-
-Version 1.29.0, 2022-05-21
-``````````````````````````
-
-- Implement a workaround for the `silent failed commit
- <https://github.com/tlocke/pg8000/issues/36>`_ bug.
-
-- Previously if an empty string was sent as the query an exception would be raised, but
- that isn't done now.
-
-
-Version 1.28.3, 2022-05-18
-``````````````````````````
-
-- Put back ``__version__`` attributes that were inadvertently removed.
-
-
-Version 1.28.2, 2022-05-17
-``````````````````````````
-
-- Use a build system that's compliant with PEP517.
-
-
-Version 1.28.1, 2022-05-17
-``````````````````````````
-
-- If when doing a ``COPY FROM`` the ``stream`` parameter is an iterator of ``str``,
- pg8000 used to silently append a newline to the end. That no longer happens.
-
-
-Version 1.28.0, 2022-05-17
-``````````````````````````
-
-- When using the ``COPY FROM`` SQL statement, allow the ``stream`` parameter to be an
- iterable.
-
-
-Version 1.27.1, 2022-05-16
-``````````````````````````
-
-- The ``seconds`` attribute of ``PGInterval`` is now always a ``float``, to cope with
- fractional seconds.
-
-- Updated the ``interval`` parsers for ``iso_8601`` and ``sql_standard`` to take
- account of fractional seconds.
-
-
-Version 1.27.0, 2022-05-16
-``````````````````````````
-
-- It used to be that by default, if pg8000 received an ``interval`` type from the server
- and it was too big to fit into a ``datetime.timedelta`` then an exception would be
- raised. Now if an interval is too big for ``datetime.timedelta`` a ``PGInterval`` is
- returned.
-
-* pg8000 now supports all the output formats for an ``interval`` (``postgres``,
- ``postgres_verbose``, ``iso_8601`` and ``sql_standard``).
-
-
-Version 1.26.1, 2022-04-23
-``````````````````````````
-
-- Make sure all tests are run by the GitHub Actions tests on commit.
-- Remove support for Python 3.6
-- Remove support for PostgreSQL 9.6
-
-
-Version 1.26.0, 2022-04-18
-``````````````````````````
-
-- When connecting, raise an ``InterfaceError('network error')`` rather than let the
- underlying ``struct.error`` float up.
-
-- Make licence text the same as that used by the OSI. Previously the licence wording
- differed slightly from the BSD 3 Clause licence at
- https://opensource.org/licenses/BSD-3-Clause. This meant that automated tools didn't
- pick it up as being Open Source. The changes are believed to not alter the meaning of the license at all.
-
-
-Version 1.25.0, 2022-04-17
-``````````````````````````
-
-- Fix more cases where a ``ResourceWarning`` would be raise because of a socket that had
- been left open.
-
-- We now have a single ``InterfaceError`` with the message 'network error' for all
- network errors, with the underlying exception held in the ``cause`` of the exception.
-
-
-Version 1.24.2, 2022-04-15
-``````````````````````````
-
-- To prevent a ``ResourceWarning`` close socket if a connection can't be created.
-
-
-Version 1.24.1, 2022-03-02
-``````````````````````````
-
-- Return pg +/-infinity dates as ``str``. Previously +/-infinity pg values would cause
- an error when returned, but now we return +/-infinity as strings.
-
-
-Version 1.24.0, 2022-02-06
-``````````````````````````
-
-- Add SQL escape functions identifier() and literal() to the native API. For use when a
- query can't be parameterised and the SQL string has to be created using untrusted
- values.
-
-
-Version 1.23.0, 2021-11-13
-``````````````````````````
-
-- If a query has no parameters, then the query will no longer be parsed. Although there
- are performance benefits for doing this, the main reason is to avoid query rewriting,
- which can introduce errors.
-
-
-Version 1.22.1, 2021-11-10
-``````````````````````````
-
-- Fix bug in PGInterval type where ``str()`` failed for a millennia value.
-
-
-Version 1.22.0, 2021-10-13
-``````````````````````````
-
-- Rather than specifying the oids in the ``Parse`` step of the Postgres protocol, pg8000
- now omits them, and so Postgres will use the oids it determines from the query. This
- makes the pg8000 code simpler and also it should also make the nuances of type
- matching more straightforward.
+`Release Notes <docs/release_notes.rst>`_
diff --git a/contrib/python/pg8000/pg8000/core.py b/contrib/python/pg8000/pg8000/core.py
index 3e55957755..8375b95213 100644
--- a/contrib/python/pg8000/pg8000/core.py
+++ b/contrib/python/pg8000/pg8000/core.py
@@ -28,7 +28,7 @@ def pack_funcs(fmt):
i_pack, i_unpack = pack_funcs("i")
-h_pack, h_unpack = pack_funcs("h")
+H_pack, H_unpack = pack_funcs("H")
ii_pack, ii_unpack = pack_funcs("ii")
ihihih_pack, ihihih_unpack = pack_funcs("ihihih")
ci_pack, ci_unpack = pack_funcs("ci")
@@ -623,7 +623,7 @@ class CoreConnection:
self._backend_key_data = data
def handle_ROW_DESCRIPTION(self, data, context):
- count = h_unpack(data)[0]
+ count = H_unpack(data)[0]
idx = 2
columns = []
input_funcs = []
@@ -656,7 +656,7 @@ class CoreConnection:
def send_PARSE(self, statement_name_bin, statement, oids=()):
val = bytearray(statement_name_bin)
val.extend(statement.encode(self._client_encoding) + NULL_BYTE)
- val.extend(h_pack(len(oids)))
+ val.extend(H_pack(len(oids)))
for oid in oids:
val.extend(i_pack(0 if oid == -1 else oid))
@@ -762,7 +762,7 @@ class CoreConnection:
"""https://www.postgresql.org/docs/current/protocol-message-formats.html"""
retval = bytearray(
- NULL_BYTE + statement_name_bin + h_pack(0) + h_pack(len(params))
+ NULL_BYTE + statement_name_bin + H_pack(0) + H_pack(len(params))
)
for value in params:
@@ -772,7 +772,7 @@ class CoreConnection:
val = value.encode(self._client_encoding)
retval.extend(i_pack(len(val)))
retval.extend(val)
- retval.extend(h_pack(0))
+ retval.extend(H_pack(0))
self._send_message(BIND, retval)
_write(self._sock, FLUSH_MSG)
diff --git a/contrib/python/pg8000/ya.make b/contrib/python/pg8000/ya.make
index 5067622704..f04dfe2320 100644
--- a/contrib/python/pg8000/ya.make
+++ b/contrib/python/pg8000/ya.make
@@ -2,7 +2,7 @@
PY3_LIBRARY()
-VERSION(1.30.4)
+VERSION(1.30.5)
LICENSE(BSD-3-Clause)
diff --git a/yt/yt/core/misc/optional.h b/library/cpp/yt/misc/optional.h
index cfae5c36d5..cfae5c36d5 100644
--- a/yt/yt/core/misc/optional.h
+++ b/library/cpp/yt/misc/optional.h
diff --git a/yt/yt/core/concurrency/fair_share_action_queue-inl.h b/yt/yt/core/concurrency/fair_share_action_queue-inl.h
index 11cf50e4eb..5c3f95dc0e 100644
--- a/yt/yt/core/concurrency/fair_share_action_queue-inl.h
+++ b/yt/yt/core/concurrency/fair_share_action_queue-inl.h
@@ -16,8 +16,9 @@ public:
TEnumIndexedFairShareActionQueue(
const TString& threadName,
const std::vector<TString>& queueNames,
- const THashMap<TString, std::vector<TString>>& queueToBucket)
- : Queue_(CreateFairShareActionQueue(threadName, queueNames, queueToBucket))
+ const THashMap<TString, std::vector<TString>>& bucketToQueues,
+ NProfiling::IRegistryImplPtr registry)
+ : Queue_(CreateFairShareActionQueue(threadName, queueNames, bucketToQueues, std::move(registry)))
{ }
const IInvokerPtr& GetInvoker(EQueue queue) override
@@ -39,21 +40,22 @@ private:
template <typename EQueue, typename EBucket>
IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueue(
const TString& threadName,
- const THashMap<EBucket, std::vector<EQueue>>& queueToBucket)
+ const THashMap<EBucket, std::vector<EQueue>>& bucketToQueues,
+ NProfiling::IRegistryImplPtr registry)
{
std::vector<TString> queueNames;
for (const auto& queueName : TEnumTraits<EQueue>::GetDomainNames()) {
queueNames.push_back(TString{queueName});
}
THashMap<TString, std::vector<TString>> stringBuckets;
- for (const auto& [bucketName, bucket] : queueToBucket) {
+ for (const auto& [bucketName, bucket] : bucketToQueues) {
auto& stringBucket = stringBuckets[ToString(bucketName)];
stringBucket.reserve(bucket.size());
for (const auto& queue : bucket) {
stringBucket.push_back(ToString(queue));
}
}
- return New<TEnumIndexedFairShareActionQueue<EQueue>>(threadName, queueNames, stringBuckets);
+ return New<TEnumIndexedFairShareActionQueue<EQueue>>(threadName, queueNames, stringBuckets, std::move(registry));
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/concurrency/fair_share_action_queue.cpp b/yt/yt/core/concurrency/fair_share_action_queue.cpp
index 21357112cd..dae24f21ba 100644
--- a/yt/yt/core/concurrency/fair_share_action_queue.cpp
+++ b/yt/yt/core/concurrency/fair_share_action_queue.cpp
@@ -29,7 +29,8 @@ public:
TFairShareActionQueue(
const TString& threadName,
const std::vector<TString>& queueNames,
- const THashMap<TString, std::vector<TString>>& queueToBucket)
+ const THashMap<TString, std::vector<TString>>& bucketToQueues,
+ NProfiling::IRegistryImplPtr registry)
: ShutdownCookie_(RegisterShutdownCallback(
Format("FairShareActionQueue(%v)", threadName),
BIND_NO_PROPAGATE(&TFairShareActionQueue::Shutdown, MakeWeak(this), /*graceful*/ false),
@@ -37,7 +38,7 @@ public:
{
THashMap<TString, int> queueNameToIndex;
for (int queueIndex = 0; queueIndex < std::ssize(queueNames); ++queueIndex) {
- YT_VERIFY(queueNameToIndex.emplace(queueNames[queueIndex], queueIndex).second);
+ EmplaceOrCrash(queueNameToIndex, queueNames[queueIndex], queueIndex);
}
QueueIndexToBucketIndex_.resize(queueNames.size(), -1);
@@ -47,18 +48,20 @@ public:
std::vector<TBucketDescription> bucketDescriptions;
THashSet<TString> createdQueues;
int nextBucketIndex = 0;
- for (const auto& [bucketName, bucketQueues] : queueToBucket) {
+ for (const auto& [bucketName, bucketQueues] : bucketToQueues) {
int bucketIndex = nextBucketIndex++;
auto& bucketDescription = bucketDescriptions.emplace_back();
- bucketDescription.BucketTagSet = GetBucketTags(threadName, bucketName);
for (int bucketQueueIndex = 0; bucketQueueIndex < std::ssize(bucketQueues); ++bucketQueueIndex) {
const auto& queueName = bucketQueues[bucketQueueIndex];
- auto queueIndex = GetOrCrash(queueNameToIndex, queueName);
- YT_VERIFY(createdQueues.insert(queueName).second);
- QueueIndexToBucketIndex_[queueIndex] = bucketIndex;
- QueueIndexToBucketQueueIndex_[queueIndex] = bucketQueueIndex;
- bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, queueName));
+ bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, bucketName, queueName));
bucketDescription.QueueProfilerTags.push_back(New<NYTProf::TProfilerTag>("queue", queueName));
+
+ {
+ int queueIndex = GetOrCrash(queueNameToIndex, queueName);
+ InsertOrCrash(createdQueues, queueName);
+ QueueIndexToBucketIndex_[queueIndex] = bucketIndex;
+ QueueIndexToBucketQueueIndex_[queueIndex] = bucketQueueIndex;
+ }
}
BucketNames_.push_back(bucketName);
}
@@ -70,13 +73,16 @@ public:
}
auto& bucketDescription = bucketDescriptions.emplace_back();
- bucketDescription.BucketTagSet = GetBucketTags(threadName, queueName);
- bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, queueName));
+ bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, queueName, queueName));
bucketDescription.QueueProfilerTags.push_back(New<NYTProf::TProfilerTag>("queue", queueName));
- auto queueIndex = GetOrCrash(queueNameToIndex, queueName);
- QueueIndexToBucketIndex_[queueIndex] = nextBucketIndex++;
- QueueIndexToBucketQueueIndex_[queueIndex] = 0;
- YT_VERIFY(createdQueues.emplace(queueName).second);
+
+ {
+ auto queueIndex = GetOrCrash(queueNameToIndex, queueName);
+ QueueIndexToBucketIndex_[queueIndex] = nextBucketIndex++;
+ QueueIndexToBucketQueueIndex_[queueIndex] = 0;
+ InsertOrCrash(createdQueues, queueName);
+ }
+
BucketNames_.push_back(queueName);
}
@@ -88,7 +94,7 @@ public:
YT_VERIFY(QueueIndexToBucketQueueIndex_[queueIndex] != -1);
}
- Queue_ = New<TFairShareInvokerQueue>(CallbackEventCount_, std::move(bucketDescriptions));
+ Queue_ = New<TFairShareInvokerQueue>(CallbackEventCount_, std::move(bucketDescriptions), std::move(registry));
Thread_ = New<TFairShareQueueSchedulerThread>(Queue_, CallbackEventCount_, threadName, threadName);
}
@@ -171,9 +177,10 @@ private:
IFairShareActionQueuePtr CreateFairShareActionQueue(
const TString& threadName,
const std::vector<TString>& queueNames,
- const THashMap<TString, std::vector<TString>>& queueToBucket)
+ const THashMap<TString, std::vector<TString>>& bucketToQueues,
+ NProfiling::IRegistryImplPtr registry)
{
- return New<TFairShareActionQueue>(threadName, queueNames, queueToBucket);
+ return New<TFairShareActionQueue>(threadName, queueNames, bucketToQueues, std::move(registry));
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/concurrency/fair_share_action_queue.h b/yt/yt/core/concurrency/fair_share_action_queue.h
index f0d69caa50..33ac86a5ad 100644
--- a/yt/yt/core/concurrency/fair_share_action_queue.h
+++ b/yt/yt/core/concurrency/fair_share_action_queue.h
@@ -6,6 +6,8 @@
#include <yt/yt/core/misc/range.h>
+#include <yt/yt/core/profiling/public.h>
+
namespace NYT::NConcurrency {
////////////////////////////////////////////////////////////////////////////////
@@ -25,7 +27,8 @@ DEFINE_REFCOUNTED_TYPE(IFairShareActionQueue)
IFairShareActionQueuePtr CreateFairShareActionQueue(
const TString& threadName,
const std::vector<TString>& queueNames,
- const THashMap<TString, std::vector<TString>>& queueToBucket = {});
+ const THashMap<TString, std::vector<TString>>& bucketToQueues = {},
+ NProfiling::IRegistryImplPtr registry = {});
////////////////////////////////////////////////////////////////////////////////
@@ -43,7 +46,8 @@ struct IEnumIndexedFairShareActionQueue
template <typename EQueue, typename EBucket = EQueue>
IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueue(
const TString& threadName,
- const THashMap<EBucket, std::vector<EQueue>>& queueToBucket = {});
+ const THashMap<EBucket, std::vector<EQueue>>& bucketToQueues = {},
+ NProfiling::IRegistryImplPtr registry = {});
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/concurrency/fair_share_invoker_queue.cpp b/yt/yt/core/concurrency/fair_share_invoker_queue.cpp
index 099cc8aa52..e42cdc16a6 100644
--- a/yt/yt/core/concurrency/fair_share_invoker_queue.cpp
+++ b/yt/yt/core/concurrency/fair_share_invoker_queue.cpp
@@ -16,7 +16,8 @@ using namespace NYTree;
TFairShareInvokerQueue::TFairShareInvokerQueue(
TIntrusivePtr<NThreading::TEventCount> callbackEventCount,
- const std::vector<TBucketDescription>& bucketDescriptions)
+ const std::vector<TBucketDescription>& bucketDescriptions,
+ NProfiling::IRegistryImplPtr registry)
: Weights_(bucketDescriptions.size(), 1.0)
{
Buckets_.reserve(bucketDescriptions.size());
@@ -26,7 +27,7 @@ TFairShareInvokerQueue::TFairShareInvokerQueue(
callbackEventCount,
bucketDescription.QueueTagSets,
bucketDescription.QueueProfilerTags,
- bucketDescription.BucketTagSet);
+ registry);
int profilingTagCount = bucketDescription.QueueTagSets.size();
bucket.Invokers.reserve(profilingTagCount);
for (int index = 0; index < profilingTagCount; ++index) {
diff --git a/yt/yt/core/concurrency/fair_share_invoker_queue.h b/yt/yt/core/concurrency/fair_share_invoker_queue.h
index b625c43bd6..c474ffe104 100644
--- a/yt/yt/core/concurrency/fair_share_invoker_queue.h
+++ b/yt/yt/core/concurrency/fair_share_invoker_queue.h
@@ -21,7 +21,6 @@ struct TBucketDescription
{
std::vector<NProfiling::TTagSet> QueueTagSets;
std::vector<NYTProf::TProfilerTagPtr> QueueProfilerTags;
- NProfiling::TTagSet BucketTagSet;
};
////////////////////////////////////////////////////////////////////////////////
@@ -32,7 +31,8 @@ class TFairShareInvokerQueue
public:
TFairShareInvokerQueue(
TIntrusivePtr<NThreading::TEventCount> callbackEventCount,
- const std::vector<TBucketDescription>& bucketDescriptions);
+ const std::vector<TBucketDescription>& bucketDescriptions,
+ NProfiling::IRegistryImplPtr registry = {});
~TFairShareInvokerQueue();
diff --git a/yt/yt/core/concurrency/invoker_queue.cpp b/yt/yt/core/concurrency/invoker_queue.cpp
index 5c0607e6dd..fec32d6c86 100644
--- a/yt/yt/core/concurrency/invoker_queue.cpp
+++ b/yt/yt/core/concurrency/invoker_queue.cpp
@@ -287,10 +287,11 @@ private:
template <class TQueueImpl>
TInvokerQueue<TQueueImpl>::TInvokerQueue(
TIntrusivePtr<NThreading::TEventCount> callbackEventCount,
- const TTagSet& counterTagSet)
+ const TTagSet& counterTagSet,
+ NProfiling::IRegistryImplPtr registry)
: CallbackEventCount_(std::move(callbackEventCount))
{
- Counters_.push_back(CreateCounters(counterTagSet));
+ Counters_.push_back(CreateCounters(counterTagSet, std::move(registry)));
}
template <class TQueueImpl>
@@ -298,18 +299,16 @@ TInvokerQueue<TQueueImpl>::TInvokerQueue(
TIntrusivePtr<NThreading::TEventCount> callbackEventCount,
const std::vector<TTagSet>& counterTagSets,
const std::vector<NYTProf::TProfilerTagPtr>& profilerTags,
- const TTagSet& cumulativeCounterTagSet)
+ NProfiling::IRegistryImplPtr registry)
: CallbackEventCount_(std::move(callbackEventCount))
{
YT_VERIFY(counterTagSets.size() == profilerTags.size());
Counters_.reserve(counterTagSets.size());
for (const auto& tagSet : counterTagSets) {
- Counters_.push_back(CreateCounters(tagSet));
+ Counters_.push_back(CreateCounters(tagSet, registry));
}
- CumulativeCounters_ = CreateCounters(cumulativeCounterTagSet);
-
ProfilingTagSettingInvokers_.reserve(Counters_.size());
for (int index = 0; index < std::ssize(Counters_); ++index) {
ProfilingTagSettingInvokers_.push_back(
@@ -388,14 +387,10 @@ TCpuInstant TInvokerQueue<TQueueImpl>::EnqueueCallback(
auto action = MakeAction(std::move(callback), profilingTag, std::move(profilerTag), cpuInstant);
- auto updateCounters = [&] (const TCountersPtr& counters) {
- if (counters) {
- counters->ActiveCallbacks += 1;
- counters->EnqueuedCounter.Increment();
- }
- };
- updateCounters(Counters_[profilingTag]);
- updateCounters(CumulativeCounters_);
+ if (Counters_[profilingTag]) {
+ Counters_[profilingTag]->ActiveCallbacks += 1;
+ Counters_[profilingTag]->EnqueuedCounter.Increment();
+ }
QueueImpl_.Enqueue(std::move(action));
return cpuInstant;
@@ -421,14 +416,10 @@ TCpuInstant TInvokerQueue<TQueueImpl>::EnqueueCallbacks(
actions.push_back(MakeAction(std::move(callback), profilingTag, profilerTag, cpuInstant));
}
- auto updateCounters = [&] (const TCountersPtr& counters) {
- if (counters) {
- counters->ActiveCallbacks += std::ssize(actions);
- counters->EnqueuedCounter.Increment(std::ssize(actions));
- }
- };
- updateCounters(Counters_[profilingTag]);
- updateCounters(CumulativeCounters_);
+ if (Counters_[profilingTag]) {
+ Counters_[profilingTag]->ActiveCallbacks += std::ssize(actions);
+ Counters_[profilingTag]->EnqueuedCounter.Increment(std::ssize(actions));
+ }
QueueImpl_.Enqueue(actions);
return cpuInstant;
@@ -493,14 +484,10 @@ bool TInvokerQueue<TQueueImpl>::BeginExecute(TEnqueuedAction* action, typename T
WaitTimeObserver_(waitTime);
}
- auto updateCounters = [&] (const TCountersPtr& counters) {
- if (counters) {
- counters->DequeuedCounter.Increment();
- counters->WaitTimer.Record(waitTime);
- }
- };
- updateCounters(Counters_[action->ProfilingTag]);
- updateCounters(CumulativeCounters_);
+ if (Counters_[action->ProfilingTag]) {
+ Counters_[action->ProfilingTag]->DequeuedCounter.Increment();
+ Counters_[action->ProfilingTag]->WaitTimer.Record(waitTime);
+ }
if (const auto& profilerTag = action->ProfilerTag) {
GetTlsRef(CpuProfilerTagGuard) = TCpuProfilerTagGuard(profilerTag);
@@ -532,16 +519,12 @@ void TInvokerQueue<TQueueImpl>::EndExecute(TEnqueuedAction* action)
auto timeFromStart = CpuDurationToDuration(action->FinishedAt - action->StartedAt);
auto timeFromEnqueue = CpuDurationToDuration(action->FinishedAt - action->EnqueuedAt);
- auto updateCounters = [&] (const TCountersPtr& counters) {
- if (counters) {
- counters->ExecTimer.Record(timeFromStart);
- counters->CumulativeTimeCounter.Add(timeFromStart);
- counters->TotalTimer.Record(timeFromEnqueue);
- counters->ActiveCallbacks -= 1;
- }
- };
- updateCounters(Counters_[action->ProfilingTag]);
- updateCounters(CumulativeCounters_);
+ if (Counters_[action->ProfilingTag]) {
+ Counters_[action->ProfilingTag]->ExecTimer.Record(timeFromStart);
+ Counters_[action->ProfilingTag]->CumulativeTimeCounter.Add(timeFromStart);
+ Counters_[action->ProfilingTag]->TotalTimer.Record(timeFromEnqueue);
+ Counters_[action->ProfilingTag]->ActiveCallbacks -= 1;
+ }
}
template <class TQueueImpl>
@@ -586,9 +569,9 @@ void TInvokerQueue<TQueueImpl>::RegisterWaitTimeObserver(TWaitTimeObserver waitT
}
template <class TQueueImpl>
-typename TInvokerQueue<TQueueImpl>::TCountersPtr TInvokerQueue<TQueueImpl>::CreateCounters(const TTagSet& tagSet)
+typename TInvokerQueue<TQueueImpl>::TCountersPtr TInvokerQueue<TQueueImpl>::CreateCounters(const TTagSet& tagSet, NProfiling::IRegistryImplPtr registry)
{
- auto profiler = TProfiler("/action_queue").WithTags(tagSet).WithHot();
+ auto profiler = TProfiler(registry, "/action_queue").WithTags(tagSet).WithHot();
auto counters = std::make_unique<TCounters>();
counters->EnqueuedCounter = profiler.Counter("/enqueued");
diff --git a/yt/yt/core/concurrency/invoker_queue.h b/yt/yt/core/concurrency/invoker_queue.h
index 556a2ea888..48b86df069 100644
--- a/yt/yt/core/concurrency/invoker_queue.h
+++ b/yt/yt/core/concurrency/invoker_queue.h
@@ -127,13 +127,14 @@ class TInvokerQueue
public:
TInvokerQueue(
TIntrusivePtr<NThreading::TEventCount> callbackEventCount,
- const NProfiling::TTagSet& counterTagSet);
+ const NProfiling::TTagSet& counterTagSet,
+ NProfiling::IRegistryImplPtr registry = nullptr);
TInvokerQueue(
TIntrusivePtr<NThreading::TEventCount> callbackEventCount,
const std::vector<NProfiling::TTagSet>& counterTagSets,
const std::vector<NYTProf::TProfilerTagPtr>& profilerTags,
- const NProfiling::TTagSet& cumulativeCounterTagSet);
+ NProfiling::IRegistryImplPtr registry = nullptr);
void SetThreadId(NThreading::TThreadId threadId);
@@ -204,14 +205,13 @@ private:
using TCountersPtr = std::unique_ptr<TCounters>;
std::vector<TCountersPtr> Counters_;
- TCountersPtr CumulativeCounters_;
std::vector<IInvokerPtr> ProfilingTagSettingInvokers_;
std::atomic<bool> IsWaitTimeObserverSet_;
TWaitTimeObserver WaitTimeObserver_;
- TCountersPtr CreateCounters(const NProfiling::TTagSet& tagSet);
+ TCountersPtr CreateCounters(const NProfiling::TTagSet& tagSet, NProfiling::IRegistryImplPtr registry);
};
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/concurrency/profiling_helpers.cpp b/yt/yt/core/concurrency/profiling_helpers.cpp
index 4eb9c44c43..b6b67994c0 100644
--- a/yt/yt/core/concurrency/profiling_helpers.cpp
+++ b/yt/yt/core/concurrency/profiling_helpers.cpp
@@ -28,11 +28,13 @@ TTagSet GetBucketTags(
TTagSet GetQueueTags(
const TString& threadName,
+ const TString& bucketName,
const TString& queueName)
{
TTagSet tags;
tags.AddTag(std::pair<TString, TString>("thread", threadName));
+ tags.AddTag(std::pair<TString, TString>("bucket", bucketName), -1);
tags.AddTag(std::pair<TString, TString>("queue", queueName), -1);
return tags;
diff --git a/yt/yt/core/concurrency/profiling_helpers.h b/yt/yt/core/concurrency/profiling_helpers.h
index a5338b32cd..53ab2f6f91 100644
--- a/yt/yt/core/concurrency/profiling_helpers.h
+++ b/yt/yt/core/concurrency/profiling_helpers.h
@@ -14,6 +14,7 @@ NProfiling::TTagSet GetBucketTags(
NProfiling::TTagSet GetQueueTags(
const TString& threadName,
+ const TString& bucketName,
const TString& queueName);
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp b/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp
new file mode 100644
index 0000000000..5f98acb3f0
--- /dev/null
+++ b/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp
@@ -0,0 +1,166 @@
+#include <yt/yt/core/test_framework/framework.h>
+
+#include <yt/yt/core/profiling/timing.h>
+
+#include <yt/yt/core/concurrency/action_queue.h>
+#include <yt/yt/core/concurrency/delayed_executor.h>
+#include <yt/yt/core/concurrency/fair_share_action_queue.h>
+#include <yt/yt/core/concurrency/scheduler.h>
+
+#include <yt/yt/core/misc/collection_helpers.h>
+
+#include <yt/yt/library/profiling/solomon/exporter.h>
+
+#include <util/datetime/base.h>
+
+#include <algorithm>
+#include <array>
+#include <ranges>
+#include <utility>
+
+namespace NYT::NConcurrency {
+namespace {
+
+using namespace NProfiling;
+
+class TTestFairShareActionQueue
+ : public ::testing::Test
+{
+public:
+ TSolomonExporterConfigPtr CreateExporterConfig()
+ {
+ auto config = New<TSolomonExporterConfig>();
+ config->GridStep = TDuration::Seconds(1);
+ config->EnableCoreProfilingCompatibility = true;
+ config->EnableSelfProfiling = false;
+
+ return config;
+ }
+
+ template <class F>
+ void SyncRunCallback(IInvokerPtr invoker, F&& func)
+ {
+ WaitFor(BIND(std::forward<F>(func)).AsyncVia(invoker).Run()).ThrowOnError();
+ }
+
+ auto GetSensors(TString json)
+ {
+ for (auto& c : json) {
+ if (c == ':') {
+ c = '=';
+ } else if (c == ',') {
+ c = ';';
+ }
+ }
+
+ auto yson = NYson::TYsonString(json);
+
+ auto list = NYTree::ConvertToNode(yson)->AsMap()->FindChild("sensors");
+
+ EXPECT_TRUE(list);
+
+ return list->AsList()->GetChildren();
+ }
+
+ template <class EQueues, class EBuckets>
+ void VerifyJson(TString json, THashMap<EBuckets, std::vector<EQueues>> bucketToQueues)
+ {
+ TEnumIndexedArray<EQueues, int> enqueuedPerQueue;
+ TEnumIndexedArray<EBuckets, int> enqueuedPerBucket;
+ int enqueuedPerThread;
+ int enqueuedTotal;
+
+ for (const auto& entry : GetSensors(json)) {
+ auto mapEntry = entry->AsMap();
+ auto labels = mapEntry->FindChild("labels")->AsMap();
+
+ auto sensor = labels->FindChildValue<TString>("sensor");
+
+ if (!sensor ||
+ sensor != "yt.action_queue.enqueued")
+ {
+ continue;
+ }
+
+ auto value = mapEntry->FindChildValue<int>("value");
+ EXPECT_TRUE(value);
+
+ if (auto threadName = labels->FindChildValue<TString>("thread")) {
+ EXPECT_EQ(threadName, "ActionQueue");
+
+ if (auto bucketName = labels->FindChildValue<TString>("bucket")) {
+ if (auto queueName = labels->FindChildValue<TString>("queue")) {
+ enqueuedPerQueue[TEnumTraits<EQueues>::FromString(*queueName)] = *value;
+ continue;
+ }
+
+ enqueuedPerBucket[TEnumTraits<EBuckets>::FromString(*bucketName)] = *value;
+ continue;
+ }
+
+ enqueuedPerThread = *value;
+ } else {
+ enqueuedTotal = *value;
+ }
+ }
+
+ EXPECT_EQ(enqueuedPerThread, enqueuedTotal);
+
+ int totalFromBuckets = 0;
+ for (auto bucketValue : enqueuedPerBucket) {
+ totalFromBuckets += bucketValue;
+ }
+ EXPECT_EQ(enqueuedPerThread, totalFromBuckets);
+
+ for (auto bucket : TEnumTraits<EBuckets>::GetDomainValues()) {
+ int totalFromBucketQueues = 0;
+
+ for (auto queue : bucketToQueues[bucket]) {
+ totalFromBucketQueues += enqueuedPerQueue[queue];
+ }
+ EXPECT_EQ(enqueuedPerBucket[bucket], totalFromBucketQueues);
+ }
+ }
+};
+
+DEFINE_ENUM(EBuckets,
+ ((Bucket1) (0))
+ ((Bucket2) (1))
+);
+
+DEFINE_ENUM(EQueues,
+ ((Queue1) (0))
+ ((Queue2) (1))
+ ((Queue3) (2))
+);
+
+TEST_F(TTestFairShareActionQueue, TestProfiling)
+{
+ auto registry = New<TSolomonRegistry>();
+
+ THashMap<EBuckets, std::vector<EQueues>> bucketToQueues{};
+ bucketToQueues[EBuckets::Bucket1] = std::vector{EQueues::Queue1, EQueues::Queue2};
+ bucketToQueues[EBuckets::Bucket2] = std::vector{EQueues::Queue3};
+ auto queue = CreateEnumIndexedFairShareActionQueue<EQueues>("ActionQueue", bucketToQueues, registry);
+
+ auto config = CreateExporterConfig();
+ auto exporter = New<TSolomonExporter>(config, registry);
+
+ exporter->Start();
+
+ for (auto value : TEnumTraits<EQueues>::GetDomainValues()) {
+ SyncRunCallback(queue->GetInvoker(value), [] {});
+ }
+
+ Sleep(TDuration::Seconds(5));
+
+ auto json = exporter->ReadJson();
+ ASSERT_TRUE(json);
+
+ exporter->Stop();
+
+ VerifyJson(*json, std::move(bucketToQueues));
+}
+
+} // namespace
+} // namespace NYT::NConcurrency
diff --git a/yt/yt/core/concurrency/unittests/ya.make b/yt/yt/core/concurrency/unittests/ya.make
index b8cad1eafd..4abc7ebc0e 100644
--- a/yt/yt/core/concurrency/unittests/ya.make
+++ b/yt/yt/core/concurrency/unittests/ya.make
@@ -17,6 +17,7 @@ SRCS(
coroutines_ut.cpp
count_down_latch_ut.cpp
delayed_executor_ut.cpp
+ fair_share_action_queue_ut.cpp
fair_share_invoker_pool_ut.cpp
fair_share_thread_pool_ut.cpp
fair_throttler_ut.cpp
diff --git a/yt/yt/core/misc/common.h b/yt/yt/core/misc/common.h
index bbc9a9ae02..a9e6aa1bb1 100644
--- a/yt/yt/core/misc/common.h
+++ b/yt/yt/core/misc/common.h
@@ -9,6 +9,10 @@
#include <library/cpp/yt/memory/new.h>
#include <library/cpp/yt/memory/ref_counted.h>
+// NB: Must be included after IntrusivePtr<T> for
+// template instantiation correctness.
+#include <library/cpp/yt/misc/optional.h>
+
#include <util/datetime/base.h>
#include <util/generic/hash.h>
diff --git a/yt/yt/core/misc/protobuf_helpers.h b/yt/yt/core/misc/protobuf_helpers.h
index fc5453ce52..893c387076 100644
--- a/yt/yt/core/misc/protobuf_helpers.h
+++ b/yt/yt/core/misc/protobuf_helpers.h
@@ -2,7 +2,6 @@
#include "guid.h"
#include "mpl.h"
-#include "optional.h"
#include "object_pool.h"
#include "range.h"
#include "serialize.h"
@@ -14,6 +13,7 @@
#include <library/cpp/yt/memory/ref.h>
+#include <library/cpp/yt/misc/optional.h>
#include <library/cpp/yt/misc/preprocessor.h>
#include <google/protobuf/message.h>
diff --git a/yt/yt/core/yson/pull_parser-inl.h b/yt/yt/core/yson/pull_parser-inl.h
index 48a0ea9203..e5ad9f8d06 100644
--- a/yt/yt/core/yson/pull_parser-inl.h
+++ b/yt/yt/core/yson/pull_parser-inl.h
@@ -6,9 +6,10 @@
#include "detail.h"
-#include <yt/yt/core/misc/optional.h>
#include <yt/yt/core/misc/parser_helpers.h>
+#include <library/cpp/yt/misc/optional.h>
+
#include <optional>
namespace NYT::NYson {
diff --git a/yt/yt/core/ytree/attributes.h b/yt/yt/core/ytree/attributes.h
index c703bfa529..0c99076f19 100644
--- a/yt/yt/core/ytree/attributes.h
+++ b/yt/yt/core/ytree/attributes.h
@@ -4,7 +4,7 @@
#include <yt/yt/core/yson/string.h>
-#include <yt/yt/core/misc/optional.h>
+#include <library/cpp/yt/misc/optional.h>
namespace NYT::NYTree {
diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h
index 046a66188d..d6750c6970 100644
--- a/yt/yt/core/ytree/yson_struct_detail.h
+++ b/yt/yt/core/ytree/yson_struct_detail.h
@@ -5,7 +5,8 @@
#include <yt/yt/core/yson/public.h>
#include <yt/yt/core/ypath/public.h>
#include <yt/yt/core/ytree/public.h>
-#include <yt/yt/core/misc/optional.h>
+
+#include <library/cpp/yt/misc/optional.h>
namespace NYT::NYTree {