diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2024-03-08 12:14:09 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2024-03-08 12:22:31 +0300 |
commit | 9584fd808b5bbcd989348a67768064134e3341d9 (patch) | |
tree | 601ec8deabb8dcdca66d4123cd7a0dcf7d804243 | |
parent | e8116f70daae60dcbf13132581dba893802ce3fe (diff) | |
download | ydb-9584fd808b5bbcd989348a67768064134e3341d9.tar.gz |
Intermediate changes
27 files changed, 318 insertions, 2240 deletions
diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index d883f6dec6..171a09eebb 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.98.9 +Version: 6.98.10 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py index 0016f2fa1a..cdce1afacf 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py @@ -40,7 +40,7 @@ import attr from hypothesis.errors import Frozen, InvalidArgument, StopTest from hypothesis.internal.cache import LRUReusedCache -from hypothesis.internal.compat import floor, int_from_bytes, int_to_bytes +from hypothesis.internal.compat import add_note, floor, int_from_bytes, int_to_bytes from hypothesis.internal.conjecture.floats import float_to_lex, lex_to_float from hypothesis.internal.conjecture.junkdrawer import IntList, uniform from hypothesis.internal.conjecture.utils import ( @@ -1748,14 +1748,17 @@ class ConjectureData: try: if not at_top_level: return strategy.do_draw(self) - else: - assert start_time is not None + assert start_time is not None + key = observe_as or f"generate:unlabeled_{len(self.draw_times)}" + try: strategy.validate() try: return strategy.do_draw(self) finally: - key = observe_as or f"generate:unlabeled_{len(self.draw_times)}" self.draw_times[key] = time.perf_counter() - start_time + except Exception as err: + add_note(err, f"while generating {key[9:]!r} from {strategy!r}") + raise finally: self.stop_example() diff --git a/contrib/python/hypothesis/py3/hypothesis/stateful.py b/contrib/python/hypothesis/py3/hypothesis/stateful.py index bf7271fc4c..2ab7ef13d5 100644 --- a/contrib/python/hypothesis/py3/hypothesis/stateful.py +++ b/contrib/python/hypothesis/py3/hypothesis/stateful.py @@ -47,6 +47,7 @@ from hypothesis._settings import ( from hypothesis.control import _current_build_context, current_build_context from hypothesis.core import TestFunc, given from hypothesis.errors import InvalidArgument, InvalidDefinition +from hypothesis.internal.compat import add_note from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.healthcheck import fail_health_check from hypothesis.internal.observability import TESTCASE_CALLBACKS @@ -439,18 +440,16 @@ class Rule: bundles = attr.ib(init=False) def __attrs_post_init__(self): - arguments = {} + self.arguments_strategies = {} bundles = [] for k, v in sorted(self.arguments.items()): assert not isinstance(v, BundleReferenceStrategy) if isinstance(v, Bundle): bundles.append(v) consume = isinstance(v, BundleConsumer) - arguments[k] = BundleReferenceStrategy(v.name, consume=consume) - else: - arguments[k] = v + v = BundleReferenceStrategy(v.name, consume=consume) + self.arguments_strategies[k] = v self.bundles = tuple(bundles) - self.arguments_strategy = st.fixed_dictionaries(arguments) self_strategy = st.runner() @@ -978,7 +977,15 @@ class RuleStrategy(SearchStrategy): .filter(lambda r: feature_flags.is_enabled(r.function.__name__)) ) - return (rule, data.draw(rule.arguments_strategy)) + arguments = {} + for k, strat in rule.arguments_strategies.items(): + try: + arguments[k] = data.draw(strat) + except Exception as err: + rname = rule.function.__name__ + add_note(err, f"while generating {k!r} from {strat!r} for rule {rname}") + raise + return (rule, arguments) def is_valid(self, rule): predicates = self.machine._observability_predicates diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py index d91102cb50..7c4e584ee6 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py @@ -37,6 +37,7 @@ from hypothesis.errors import HypothesisWarning, InvalidArgument, ResolutionFail from hypothesis.internal.compat import PYPY, BaseExceptionGroup, ExceptionGroup from hypothesis.internal.conjecture.utils import many as conjecture_utils_many from hypothesis.internal.filtering import max_len, min_len +from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies._internal.datetime import zoneinfo # type: ignore from hypothesis.strategies._internal.ipaddress import ( SPECIAL_IPv4_RANGES, @@ -300,16 +301,40 @@ def _get_constraints(args: Tuple[Any, ...]) -> Iterator["at.BaseMetadata"]: yield from at.Len(arg.start or 0, arg.stop) +def _flat_annotated_repr_parts(annotated_type): + # Helper to get a good error message in find_annotated_strategy() below. + type_reps = [ + get_pretty_function_description(a) + for a in annotated_type.__args__ + if not isinstance(a, typing.TypeVar) + ] + metadata_reps = [] + for m in getattr(annotated_type, "__metadata__", ()): + if is_annotated_type(m): + ts, ms = _flat_annotated_repr_parts(m) + type_reps.extend(ts) + metadata_reps.extend(ms) + else: + metadata_reps.append(get_pretty_function_description(m)) + return type_reps, metadata_reps + + def find_annotated_strategy(annotated_type): metadata = getattr(annotated_type, "__metadata__", ()) if any(is_annotated_type(arg) for arg in metadata): - # We are in the case where one of the metadata argument - # is itself an annotated type. Although supported at runtime, - # This shouldn't be allowed: we prefer to raise here + # Annotated[Annotated[T], ...] is perfectly acceptable, but it's all to easy + # to instead write Annotated[T1, Annotated[T2, ...]] - and nobody else checks + # for that at runtime. Once you add generics this can be seriously confusing, + # so we go to some trouble to give a helpful error message. + # For details: https://github.com/HypothesisWorks/hypothesis/issues/3891 + ty_rep = repr(annotated_type).replace("typing.Annotated", "Annotated") + ts, ms = _flat_annotated_repr_parts(annotated_type) + bits = ", ".join([" | ".join(dict.fromkeys(ts or "?")), *dict.fromkeys(ms)]) raise ResolutionFailed( - f"Failed to resolve strategy for the following Annotated type: {annotated_type}." - "Arguments to the Annotated type cannot be Annotated." + f"`{ty_rep}` is invalid because nesting Annotated is only allowed for " + f"the first (type) argument, not for later (metadata) arguments. " + f"Did you mean `Annotated[{bits}]`?" ) for arg in reversed(metadata): if isinstance(arg, st.SearchStrategy): diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index f54f8dc42f..ce617c47e9 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 98, 9) +__version_info__ = (6, 98, 10) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index 89269430da..4c7de0c00e 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.98.9) +VERSION(6.98.10) LICENSE(MPL-2.0) diff --git a/contrib/python/pg8000/.dist-info/METADATA b/contrib/python/pg8000/.dist-info/METADATA index 93aec222db..291f74df6f 100644 --- a/contrib/python/pg8000/.dist-info/METADATA +++ b/contrib/python/pg8000/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pg8000 -Version: 1.30.4 +Version: 1.30.5 Summary: PostgreSQL interface library License: BSD 3-Clause License Project-URL: Homepage, https://github.com/tlocke/pg8000 @@ -1351,839 +1351,13 @@ pg8000 encounters a network problem it'll raise an ``InterfaceError`` with the m Native API Docs --------------- -pg8000.native.Error -``````````````````` - -Generic exception that is the base exception of the other error exceptions. - - -pg8000.native.InterfaceError -```````````````````````````` - -For errors that originate within pg8000. - - -pg8000.native.DatabaseError -``````````````````````````` - -For errors that originate from the server. - -pg8000.native.Connection(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None) -`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` - -Creates a connection to a PostgreSQL database. - -user - The username to connect to the PostgreSQL server with. If your server character - encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes, - eg. ``'my_name'.encode('EUC-JP')``. - -host - The hostname of the PostgreSQL server to connect with. Providing this parameter is - necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be - provided. The default is ``localhost``. - -database - The name of the database instance to connect with. If ``None`` then the PostgreSQL - server will assume the database name is the same as the username. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database`` - as bytes, eg. ``'my_db'.encode('EUC-JP')``. - -port - The TCP/IP port of the PostgreSQL server instance. This parameter defaults to - ``5432``, the registered common port of PostgreSQL TCP/IP servers. - -password - The user password to connect to the server with. This parameter is optional; if - omitted and the database server requests password-based authentication, the connection - will fail to open. If this parameter is provided but not - requested by the server, no error will occur. - - If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``. - -source_address - The source IP address which initiates the connection to the PostgreSQL server. The - default is ``None`` which means that the operating system will choose the source - address. - -unix_sock - The path to the UNIX socket to access the database through, for example, - ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided. - -ssl_context - This governs SSL encryption for TCP/IP sockets. It can have three values: - - - ``None``, meaning no SSL (the default) - - - ``True``, means use SSL with an |ssl.SSLContext|_ created using - |ssl.create_default_context()|_ - - - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection. - - If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific - attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL - socket, but not to request SSL from the PostgreSQL server. Note that this means you - can't use SCRAM authentication with channel binding. - -timeout - This is the time in seconds before the connection to the server will time out. The - default is ``None`` which means no timeout. - -tcp_keepalive - If ``True`` then use `TCP keepalive - <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``. - -application_name - Sets the `application_name - <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_. - If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default - is ``None`` which means that the server will set the application name. - -replication - Used to run in `streaming replication mode - <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide values as - bytes, eg. ``'database'.encode('EUC-JP')``. - -sock - A socket-like object to use for the connection. For example, ``sock`` could be a plain - ``socket.socket``, or it could represent an SSH tunnel or perhaps an - ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be - used to attempt to create an SSL socket from the provided socket. - -pg8000.native.Connection.notifications -`````````````````````````````````````` - -A deque of server-side `notifications -<https://www.postgresql.org/docs/current/sql-notify.html>`__ received by this database -connection (via the ``LISTEN`` / ``NOTIFY`` PostgreSQL commands). Each list item is a -three-element tuple containing the PostgreSQL backend PID that issued the notify, the -channel and the payload. - - -pg8000.native.Connection.notices -```````````````````````````````` - -A deque of server-side notices received by this database connection. - - -pg8000.native.Connection.parameter_statuses -``````````````````````````````````````````` - -A deque of server-side parameter statuses received by this database connection. - - -pg8000.native.Connection.run(sql, stream=None, types=None, \*\*kwargs) -`````````````````````````````````````````````````````````````````````` - -Executes an sql statement, and returns the results as a ``list``. For example:: - - con.run("SELECT * FROM cities where population > :pop", pop=10000) - -sql - The SQL statement to execute. Parameter placeholders appear as a ``:`` followed by the - parameter name. - -stream - For use with the PostgreSQL `COPY - <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. The nature - of the parameter depends on whether the SQL command is ``COPY FROM`` or ``COPY TO``. - - ``COPY FROM`` - The stream parameter must be a readable file-like object or an iterable. If it's an - iterable then the items can be ``str`` or binary. - ``COPY TO`` - The stream parameter must be a writable file-like object. - -types - A dictionary of oids. A key corresponds to a parameter. - -kwargs - The parameters of the SQL statement. - - -pg8000.native.Connection.row_count -`````````````````````````````````` - -This read-only attribute contains the number of rows that the last ``run()`` method -produced (for query statements like ``SELECT``) or affected (for modification statements -like ``UPDATE``. - -The value is -1 if: - -- No ``run()`` method has been performed yet. -- There was no rowcount associated with the last ``run()``. - - -pg8000.native.Connection.columns -```````````````````````````````` - -A list of column metadata. Each item in the list is a dictionary with the following -keys: - -- name -- table_oid -- column_attrnum -- type_oid -- type_size -- type_modifier -- format - - -pg8000.native.Connection.close() -```````````````````````````````` - -Closes the database connection. - - -pg8000.native.Connection.register_out_adapter(typ, out_func) -```````````````````````````````````````````````````````````` - -Register a type adapter for types going out from pg8000 to the server. - -typ - The Python class that the adapter is for. - -out_func - A function that takes the Python object and returns its string representation - in the format that the server requires. - - -pg8000.native.Connection.register_in_adapter(oid, in_func) -`````````````````````````````````````````````````````````` - -Register a type adapter for types coming in from the server to pg8000. - -oid - The PostgreSQL type identifier found in the `pg_type system catalog - <https://www.postgresql.org/docs/current/catalog-pg-type.html>`_. - -in_func - A function that takes the PostgreSQL string representation and returns a corresponding - Python object. - - -pg8000.native.Connection.prepare(sql) -````````````````````````````````````` - -Returns a ``PreparedStatement`` object which represents a `prepared statement -<https://www.postgresql.org/docs/current/sql-prepare.html>`_ on the server. It can -subsequently be repeatedly executed. - -sql - The SQL statement to prepare. Parameter placeholders appear as a ``:`` followed by the - parameter name. - - -pg8000.native.PreparedStatement -``````````````````````````````` - -A prepared statement object is returned by the ``pg8000.native.Connection.prepare()`` -method of a connection. It has the following methods: - - -pg8000.native.PreparedStatement.run(\*\*kwargs) -``````````````````````````````````````````````` - -Executes the prepared statement, and returns the results as a ``tuple``. - -kwargs - The parameters of the prepared statement. - - -pg8000.native.PreparedStatement.close() -``````````````````````````````````````` - -Closes the prepared statement, releasing the prepared statement held on the server. - - -pg8000.native.identifier(ident) -``````````````````````````````` - -Correctly quotes and escapes a string to be used as an `SQL identifier -<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS>`_. - -ident - The ``str`` to be used as an SQL identifier. - - -pg8000.native.literal(value) -```````````````````````````` - -Correctly quotes and escapes a value to be used as an `SQL literal -<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS>`_. - -value - The value to be used as an SQL literal. +`Native API Docs <docs/native_api_docs.rst>`_ DB-API 2 Docs ------------- - -Properties -`````````` - - -pg8000.dbapi.apilevel -::::::::::::::::::::: - -The DBAPI level supported, currently "2.0". - - -pg8000.dbapi.threadsafety -::::::::::::::::::::::::: - -Integer constant stating the level of thread safety the DBAPI interface supports. For -pg8000, the threadsafety value is 1, meaning that threads may share the module but not -connections. - - -pg8000.dbapi.paramstyle -::::::::::::::::::::::: - -String property stating the type of parameter marker formatting expected by -the interface. This value defaults to "format", in which parameters are -marked in this format: "WHERE name=%s". - -As an extension to the DBAPI specification, this value is not constant; it can be -changed to any of the following values: - -qmark - Question mark style, eg. ``WHERE name=?`` - -numeric - Numeric positional style, eg. ``WHERE name=:1`` - -named - Named style, eg. ``WHERE name=:paramname`` - -format - printf format codes, eg. ``WHERE name=%s`` - -pyformat - Python format codes, eg. ``WHERE name=%(paramname)s`` - - -pg8000.dbapi.STRING -::::::::::::::::::: - -String type oid. - -pg8000.dbapi.BINARY -::::::::::::::::::: - - -pg8000.dbapi.NUMBER -::::::::::::::::::: - -Numeric type oid. - - -pg8000.dbapi.DATETIME -::::::::::::::::::::: - -Timestamp type oid - - -pg8000.dbapi.ROWID -:::::::::::::::::: - -ROWID type oid - - -Functions -````````` - -pg8000.dbapi.connect(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None) -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -Creates a connection to a PostgreSQL database. - -user - The username to connect to the PostgreSQL server with. If your server character - encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes, - eg. ``'my_name'.encode('EUC-JP')``. - -host - The hostname of the PostgreSQL server to connect with. Providing this parameter is - necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be - provided. The default is ``localhost``. - -database - The name of the database instance to connect with. If ``None`` then the PostgreSQL - server will assume the database name is the same as the username. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database`` - as bytes, eg. ``'my_db'.encode('EUC-JP')``. - -port - The TCP/IP port of the PostgreSQL server instance. This parameter defaults to - ``5432``, the registered common port of PostgreSQL TCP/IP servers. - -password - The user password to connect to the server with. This parameter is optional; if - omitted and the database server requests password-based authentication, the - connection will fail to open. If this parameter is provided but not requested by the - server, no error will occur. - - If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``. - -source_address - The source IP address which initiates the connection to the PostgreSQL server. The - default is ``None`` which means that the operating system will choose the source - address. - -unix_sock - The path to the UNIX socket to access the database through, for example, - ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided. - -ssl_context - This governs SSL encryption for TCP/IP sockets. It can have three values: - - - ``None``, meaning no SSL (the default) - - ``True``, means use SSL with an |ssl.SSLContext|_ created using - |ssl.create_default_context()|_. - - - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection. - - If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific - attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL - socket, but not to request SSL from the PostgreSQL server. Note that this means you - can't use SCRAM authentication with channel binding. - -timeout - This is the time in seconds before the connection to the server will time out. The - default is ``None`` which means no timeout. - -tcp_keepalive - If ``True`` then use `TCP keepalive - <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``. - -application_name - Sets the `application_name - <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_. If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default - is ``None`` which means that the server will set the application name. - -replication - Used to run in `streaming replication mode - <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide values as - bytes, eg. ``'database'.encode('EUC-JP')``. - -sock - A socket-like object to use for the connection. For example, ``sock`` could be a plain - ``socket.socket``, or it could represent an SSH tunnel or perhaps an - ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be - used to attempt to create an SSL socket from the provided socket. - - -pg8000.dbapi.Date(year, month, day) - -Construct an object holding a date value. - -This property is part of the `DBAPI 2.0 specification -<http://www.python.org/dev/peps/pep-0249/>`_. - -Returns: `datetime.date` - - -pg8000.dbapi.Time(hour, minute, second) -::::::::::::::::::::::::::::::::::::::: - -Construct an object holding a time value. - -Returns: ``datetime.time`` - - -pg8000.dbapi.Timestamp(year, month, day, hour, minute, second) -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -Construct an object holding a timestamp value. - -Returns: ``datetime.datetime`` - - -pg8000.dbapi.DateFromTicks(ticks) -::::::::::::::::::::::::::::::::: - -Construct an object holding a date value from the given ticks value (number of seconds -since the epoch). - -Returns: ``datetime.datetime`` - - -pg8000.dbapi.TimeFromTicks(ticks) -::::::::::::::::::::::::::::::::: - -Construct an object holding a time value from the given ticks value (number of seconds -since the epoch). - -Returns: ``datetime.time`` - - -pg8000.dbapi.TimestampFromTicks(ticks) -:::::::::::::::::::::::::::::::::::::: - -Construct an object holding a timestamp value from the given ticks value (number of -seconds since the epoch). - -Returns: ``datetime.datetime`` - - -pg8000.dbapi.Binary(value) -:::::::::::::::::::::::::: - -Construct an object holding binary data. - -Returns: ``bytes``. - - -Generic Exceptions -`````````````````` - -Pg8000 uses the standard DBAPI 2.0 exception tree as "generic" exceptions. Generally, -more specific exception types are raised; these specific exception types are derived -from the generic exceptions. - -pg8000.dbapi.Warning -:::::::::::::::::::: - -Generic exception raised for important database warnings like data truncations. This -exception is not currently used by pg8000. - - -pg8000.dbapi.Error -:::::::::::::::::: - -Generic exception that is the base exception of all other error exceptions. - - -pg8000.dbapi.InterfaceError -::::::::::::::::::::::::::: - -Generic exception raised for errors that are related to the database interface rather -than the database itself. For example, if the interface attempts to use an SSL -connection but the server refuses, an InterfaceError will be raised. - - -pg8000.dbapi.DatabaseError -:::::::::::::::::::::::::: - -Generic exception raised for errors that are related to the database. This exception is -currently never raised by pg8000. - - -pg8000.dbapi.DataError -:::::::::::::::::::::: - -Generic exception raised for errors that are due to problems with the processed data. -This exception is not currently raised by pg8000. - - -pg8000.dbapi.OperationalError -::::::::::::::::::::::::::::: - -Generic exception raised for errors that are related to the database's operation and not -necessarily under the control of the programmer. This exception is currently never -raised by pg8000. - - -pg8000.dbapi.IntegrityError -::::::::::::::::::::::::::: - -Generic exception raised when the relational integrity of the database is affected. This -exception is not currently raised by pg8000. - - -pg8000.dbapi.InternalError -:::::::::::::::::::::::::: - -Generic exception raised when the database encounters an internal error. This is -currently only raised when unexpected state occurs in the pg8000 interface itself, and -is typically the result of a interface bug. - - -pg8000.dbapi.ProgrammingError -::::::::::::::::::::::::::::: - -Generic exception raised for programming errors. For example, this exception is raised -if more parameter fields are in a query string than there are available parameters. - - -pg8000.dbapi.NotSupportedError -:::::::::::::::::::::::::::::: - -Generic exception raised in case a method or database API was used which is not -supported by the database. - - -Classes -``````` - - -pg8000.dbapi.Connection -::::::::::::::::::::::: - -A connection object is returned by the ``pg8000.connect()`` function. It represents a -single physical connection to a PostgreSQL database. - - -pg8000.dbapi.Connection.autocommit -:::::::::::::::::::::::::::::::::: - -Following the DB-API specification, autocommit is off by default. It can be turned on by -setting this boolean pg8000-specific autocommit property to ``True``. - - -pg8000.dbapi.Connection.close() -::::::::::::::::::::::::::::::: - -Closes the database connection. - - -pg8000.dbapi.Connection.cursor() -:::::::::::::::::::::::::::::::: - -Creates a ``pg8000.dbapi.Cursor`` object bound to this connection. - - -pg8000.dbapi.Connection.rollback() -:::::::::::::::::::::::::::::::::: - -Rolls back the current database transaction. - - -pg8000.dbapi.Connection.tpc_begin(xid) -:::::::::::::::::::::::::::::::::::::: - -Begins a TPC transaction with the given transaction ID xid. This method should be -called outside of a transaction (i.e. nothing may have executed since the last -``commit()`` or ``rollback()``. Furthermore, it is an error to call ``commit()`` or -``rollback()`` within the TPC transaction. A ``ProgrammingError`` is raised, if the -application calls ``commit()`` or ``rollback()`` during an active TPC transaction. - - -pg8000.dbapi.Connection.tpc_commit(xid=None) -:::::::::::::::::::::::::::::::::::::::::::: - -When called with no arguments, ``tpc_commit()`` commits a TPC transaction previously -prepared with ``tpc_prepare()``. If ``tpc_commit()`` is called prior to -``tpc_prepare()``, a single phase commit is performed. A transaction manager may choose -to do this if only a single resource is participating in the global transaction. - -When called with a transaction ID ``xid``, the database commits the given transaction. -If an invalid transaction ID is provided, a ``ProgrammingError`` will be raised. This -form should be called outside of a transaction, and is intended for use in recovery. - -On return, the TPC transaction is ended. - - -pg8000.dbapi.Connection.tpc_prepare() -::::::::::::::::::::::::::::::::::::: - -Performs the first phase of a transaction started with ``.tpc_begin()``. A -``ProgrammingError`` is be raised if this method is called outside of a TPC transaction. - -After calling ``tpc_prepare()``, no statements can be executed until ``tpc_commit()`` or -``tpc_rollback()`` have been called. - - -pg8000.dbapi.Connection.tpc_recover() -::::::::::::::::::::::::::::::::::::: - -Returns a list of pending transaction IDs suitable for use with ``tpc_commit(xid)`` or -``tpc_rollback(xid)``. - - -pg8000.dbapi.Connection.tpc_rollback(xid=None) -:::::::::::::::::::::::::::::::::::::::::::::: - -When called with no arguments, ``tpc_rollback()`` rolls back a TPC transaction. It may -be called before or after ``tpc_prepare()``. - -When called with a transaction ID xid, it rolls back the given transaction. If an -invalid transaction ID is provided, a ``ProgrammingError`` is raised. This form should -be called outside of a transaction, and is intended for use in recovery. - -On return, the TPC transaction is ended. - - -pg8000.dbapi.Connection.xid(format_id, global_transaction_id, branch_qualifier) -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -Create a Transaction IDs (only global_transaction_id is used in pg) format_id and -branch_qualifier are not used in postgres global_transaction_id may be any string -identifier supported by postgres returns a tuple (format_id, global_transaction_id, -branch_qualifier) - - -pg8000.dbapi.Cursor -::::::::::::::::::: - -A cursor object is returned by the ``pg8000.dbapi.Connection.cursor()`` method of a -connection. It has the following attributes and methods: - -pg8000.dbapi.Cursor.arraysize -''''''''''''''''''''''''''''' - -This read/write attribute specifies the number of rows to fetch at a time with -``pg8000.dbapi.Cursor.fetchmany()``. It defaults to 1. - - -pg8000.dbapi.Cursor.connection -'''''''''''''''''''''''''''''' - -This read-only attribute contains a reference to the connection object (an instance of -``pg8000.dbapi.Connection``) on which the cursor was created. - - -pg8000.dbapi.Cursor.rowcount -'''''''''''''''''''''''''''' - -This read-only attribute contains the number of rows that the last ``execute()`` or -``executemany()`` method produced (for query statements like ``SELECT``) or affected -(for modification statements like ``UPDATE``. - -The value is -1 if: - -- No ``execute()`` or ``executemany()`` method has been performed yet on the cursor. - -- There was no rowcount associated with the last ``execute()``. - -- At least one of the statements executed as part of an ``executemany()`` had no row - count associated with it. - - -pg8000.dbapi.Cursor.description -''''''''''''''''''''''''''''''' - -This read-only attribute is a sequence of 7-item sequences. Each value contains -information describing one result column. The 7 items returned for each column are -(name, type_code, display_size, internal_size, precision, scale, null_ok). Only the -first two values are provided by the current implementation. - - -pg8000.dbapi.Cursor.close() -''''''''''''''''''''''''''' - -Closes the cursor. - - -pg8000.dbapi.Cursor.execute(operation, args=None, stream=None) -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Executes a database operation. Parameters may be provided as a sequence, or as a -mapping, depending upon the value of ``pg8000.dbapi.paramstyle``. Returns the cursor, -which may be iterated over. - -operation - The SQL statement to execute. - -args - If ``pg8000.dbapi.paramstyle`` is ``qmark``, ``numeric``, or ``format``, this - argument should be an array of parameters to bind into the statement. If - ``pg8000.dbapi.paramstyle`` is ``named``, the argument should be a ``dict`` mapping of - parameters. If ``pg8000.dbapi.paramstyle`` is ``pyformat``, the argument value may be - either an array or a mapping. - -stream - This is a pg8000 extension for use with the PostgreSQL `COPY - <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. For a - ``COPY FROM`` the parameter must be a readable file-like object, and for ``COPY TO`` - it must be writable. - - -pg8000.dbapi.Cursor.executemany(operation, param_sets) -'''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Prepare a database operation, and then execute it against all parameter sequences or -mappings provided. - -operation - The SQL statement to execute. - -parameter_sets - A sequence of parameters to execute the statement with. The values in the sequence - should be sequences or mappings of parameters, the same as the args argument of the - ``pg8000.dbapi.Cursor.execute()`` method. - - -pg8000.dbapi.Cursor.callproc(procname, parameters=None) -''''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Call a stored database procedure with the given name and optional parameters. - - -procname - The name of the procedure to call. - -parameters - A list of parameters. - - -pg8000.dbapi.Cursor.fetchall() -'''''''''''''''''''''''''''''' - -Fetches all remaining rows of a query result. - -Returns: A sequence, each entry of which is a sequence of field values making up a row. - - -pg8000.dbapi.Cursor.fetchmany(size=None) -'''''''''''''''''''''''''''''''''''''''' - -Fetches the next set of rows of a query result. - -size - The number of rows to fetch when called. If not provided, the - ``pg8000.dbapi.Cursor.arraysize`` attribute value is used instead. - -Returns: A sequence, each entry of which is a sequence of field values making up a row. -If no more rows are available, an empty sequence will be returned. - - -pg8000.dbapi.Cursor.fetchone() -'''''''''''''''''''''''''''''' - -Fetch the next row of a query result set. - -Returns: A row as a sequence of field values, or ``None`` if no more rows are available. - - -pg8000.dbapi.Cursor.setinputsizes(\*sizes) -'''''''''''''''''''''''''''''''''''''''''' - -Used to set the parameter types of the next query. This is useful if it's difficult for -pg8000 to work out the types from the parameters themselves (eg. for parameters of type -None). - -sizes - Positional parameters that are either the Python type of the parameter to be sent, or - the PostgreSQL oid. Common oids are available as constants such as ``pg8000.STRING``, - ``pg8000.INTEGER``, ``pg8000.TIME`` etc. - - -pg8000.dbapi.Cursor.setoutputsize(size, column=None) -'''''''''''''''''''''''''''''''''''''''''''''''''''' - -Not implemented by pg8000. - - -pg8000.dbapi.Interval -''''''''''''''''''''' - -An Interval represents a measurement of time. In PostgreSQL, an interval is defined in -the measure of months, days, and microseconds; as such, the pg8000 interval type -represents the same information. - -Note that values of the ``pg8000.dbapi.Interval.microseconds``, -``pg8000.dbapi.Interval.days``, and ``pg8000.dbapi.Interval.months`` properties are -independently measured and cannot be converted to each other. A month may be 28, 29, 30, -or 31 days, and a day may occasionally be lengthened slightly by a leap second. +`DB-API 2 Docs <docs/dbapi2_docs.rst>`_ Design Decisions @@ -2258,243 +1432,4 @@ Run ``tox`` to make sure all tests pass, then update the release notes, then do: Release Notes ------------- -Version 1.30.4, 2024-01-03 -`````````````````````````` - -- Add support for more range and multirange types. - -- Make the ``Connection.parameter_statuses`` property a ``dict`` rather than a ``dequeue``. - - -Version 1.30.3, 2023-10-31 -`````````````````````````` - -- Fix problem with PG date overflowing Python types. Now we return the ``str`` we got from the - server if we can't parse it. - - -Version 1.30.2, 2023-09-17 -`````````````````````````` - -- Bug fix where dollar-quoted string constants weren't supported. - - -Version 1.30.1, 2023-07-29 -`````````````````````````` - -- There was a problem uploading the previous version (1.30.0) to PyPI because the - markup of the README.rst was invalid. There's now a step in the automated tests to - check for this. - - -Version 1.30.0, 2023-07-27 -`````````````````````````` - -- Remove support for Python 3.7 - -- Add a ``sock`` keyword parameter for creating a connection from a pre-configured - socket. - - -Version 1.29.8, 2023-06-16 -`````````````````````````` - -- Ranges don't work with legacy API. - - -Version 1.29.7, 2023-06-16 -`````````````````````````` - -- Add support for PostgreSQL ``range`` and ``multirange`` types. Previously pg8000 - would just return them as strings, but now they're returned as ``Range`` and lists of - ``Range``. - -- The PostgreSQL ``record`` type is now returned as a ``tuple`` of strings, whereas - before it was returned as one string. - - -Version 1.29.6, 2023-05-29 -`````````````````````````` - -- Fixed two bugs with composite types. Nulls should be represented by an empty string, - and in an array of composite types, the elements should be surrounded by double - quotes. - - -Version 1.29.5, 2023-05-09 -`````````````````````````` - -- Fixed bug where pg8000 didn't handle the case when the number of bytes received from - a socket was fewer than requested. This was being interpreted as a network error, but - in fact we just needed to wait until more bytes were available. - -- When using the ``PGInterval`` type, if a response from the server contained the period - ``millennium``, it wasn't recognised. This was caused by a spelling mistake where we - had ``millenium`` rather than ``millennium``. - -- Added support for sending PostgreSQL composite types. If a value is sent as a - ``tuple``, pg8000 will send it to the server as a ``(`` delimited composite string. - - -Version 1.29.4, 2022-12-14 -`````````````````````````` - -- Fixed bug in ``pg8000.dbapi`` in the ``setinputsizes()`` method where if a ``size`` - was a recognized Python type, the method failed. - - -Version 1.29.3, 2022-10-26 -`````````````````````````` - -- Upgrade the SCRAM library to version 1.4.3. This adds support for the case where the - client supports channel binding but the server doesn't. - - -Version 1.29.2, 2022-10-09 -`````````````````````````` - -- Fixed a bug where in a literal array, items such as ``\n`` and ``\r`` weren't - escaped properly before being sent to the server. - -- Fixed a bug where if the PostgreSQL server has a half-hour time zone set, values of - type ``timestamp with time zone`` failed. This has been fixed by using the ``parse`` - function of the ``dateutil`` package if the ``datetime`` parser fails. - - -Version 1.29.1, 2022-05-23 -`````````````````````````` - -- In trying to determine if there's been a failed commit, check for ``ROLLBACK TO - SAVEPOINT``. - - -Version 1.29.0, 2022-05-21 -`````````````````````````` - -- Implement a workaround for the `silent failed commit - <https://github.com/tlocke/pg8000/issues/36>`_ bug. - -- Previously if an empty string was sent as the query an exception would be raised, but - that isn't done now. - - -Version 1.28.3, 2022-05-18 -`````````````````````````` - -- Put back ``__version__`` attributes that were inadvertently removed. - - -Version 1.28.2, 2022-05-17 -`````````````````````````` - -- Use a build system that's compliant with PEP517. - - -Version 1.28.1, 2022-05-17 -`````````````````````````` - -- If when doing a ``COPY FROM`` the ``stream`` parameter is an iterator of ``str``, - pg8000 used to silently append a newline to the end. That no longer happens. - - -Version 1.28.0, 2022-05-17 -`````````````````````````` - -- When using the ``COPY FROM`` SQL statement, allow the ``stream`` parameter to be an - iterable. - - -Version 1.27.1, 2022-05-16 -`````````````````````````` - -- The ``seconds`` attribute of ``PGInterval`` is now always a ``float``, to cope with - fractional seconds. - -- Updated the ``interval`` parsers for ``iso_8601`` and ``sql_standard`` to take - account of fractional seconds. - - -Version 1.27.0, 2022-05-16 -`````````````````````````` - -- It used to be that by default, if pg8000 received an ``interval`` type from the server - and it was too big to fit into a ``datetime.timedelta`` then an exception would be - raised. Now if an interval is too big for ``datetime.timedelta`` a ``PGInterval`` is - returned. - -* pg8000 now supports all the output formats for an ``interval`` (``postgres``, - ``postgres_verbose``, ``iso_8601`` and ``sql_standard``). - - -Version 1.26.1, 2022-04-23 -`````````````````````````` - -- Make sure all tests are run by the GitHub Actions tests on commit. -- Remove support for Python 3.6 -- Remove support for PostgreSQL 9.6 - - -Version 1.26.0, 2022-04-18 -`````````````````````````` - -- When connecting, raise an ``InterfaceError('network error')`` rather than let the - underlying ``struct.error`` float up. - -- Make licence text the same as that used by the OSI. Previously the licence wording - differed slightly from the BSD 3 Clause licence at - https://opensource.org/licenses/BSD-3-Clause. This meant that automated tools didn't - pick it up as being Open Source. The changes are believed to not alter the meaning of the license at all. - - -Version 1.25.0, 2022-04-17 -`````````````````````````` - -- Fix more cases where a ``ResourceWarning`` would be raise because of a socket that had - been left open. - -- We now have a single ``InterfaceError`` with the message 'network error' for all - network errors, with the underlying exception held in the ``cause`` of the exception. - - -Version 1.24.2, 2022-04-15 -`````````````````````````` - -- To prevent a ``ResourceWarning`` close socket if a connection can't be created. - - -Version 1.24.1, 2022-03-02 -`````````````````````````` - -- Return pg +/-infinity dates as ``str``. Previously +/-infinity pg values would cause - an error when returned, but now we return +/-infinity as strings. - - -Version 1.24.0, 2022-02-06 -`````````````````````````` - -- Add SQL escape functions identifier() and literal() to the native API. For use when a - query can't be parameterised and the SQL string has to be created using untrusted - values. - - -Version 1.23.0, 2021-11-13 -`````````````````````````` - -- If a query has no parameters, then the query will no longer be parsed. Although there - are performance benefits for doing this, the main reason is to avoid query rewriting, - which can introduce errors. - - -Version 1.22.1, 2021-11-10 -`````````````````````````` - -- Fix bug in PGInterval type where ``str()`` failed for a millennia value. - - -Version 1.22.0, 2021-10-13 -`````````````````````````` - -- Rather than specifying the oids in the ``Parse`` step of the Postgres protocol, pg8000 - now omits them, and so Postgres will use the oids it determines from the query. This - makes the pg8000 code simpler and also it should also make the nuances of type - matching more straightforward. +`Release Notes <docs/release_notes.rst>`_ diff --git a/contrib/python/pg8000/README.rst b/contrib/python/pg8000/README.rst index ad8ea650f2..55c26ddef3 100644 --- a/contrib/python/pg8000/README.rst +++ b/contrib/python/pg8000/README.rst @@ -1323,839 +1323,13 @@ pg8000 encounters a network problem it'll raise an ``InterfaceError`` with the m Native API Docs --------------- -pg8000.native.Error -``````````````````` - -Generic exception that is the base exception of the other error exceptions. - - -pg8000.native.InterfaceError -```````````````````````````` - -For errors that originate within pg8000. - - -pg8000.native.DatabaseError -``````````````````````````` - -For errors that originate from the server. - -pg8000.native.Connection(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None) -`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` - -Creates a connection to a PostgreSQL database. - -user - The username to connect to the PostgreSQL server with. If your server character - encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes, - eg. ``'my_name'.encode('EUC-JP')``. - -host - The hostname of the PostgreSQL server to connect with. Providing this parameter is - necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be - provided. The default is ``localhost``. - -database - The name of the database instance to connect with. If ``None`` then the PostgreSQL - server will assume the database name is the same as the username. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database`` - as bytes, eg. ``'my_db'.encode('EUC-JP')``. - -port - The TCP/IP port of the PostgreSQL server instance. This parameter defaults to - ``5432``, the registered common port of PostgreSQL TCP/IP servers. - -password - The user password to connect to the server with. This parameter is optional; if - omitted and the database server requests password-based authentication, the connection - will fail to open. If this parameter is provided but not - requested by the server, no error will occur. - - If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``. - -source_address - The source IP address which initiates the connection to the PostgreSQL server. The - default is ``None`` which means that the operating system will choose the source - address. - -unix_sock - The path to the UNIX socket to access the database through, for example, - ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided. - -ssl_context - This governs SSL encryption for TCP/IP sockets. It can have three values: - - - ``None``, meaning no SSL (the default) - - - ``True``, means use SSL with an |ssl.SSLContext|_ created using - |ssl.create_default_context()|_ - - - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection. - - If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific - attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL - socket, but not to request SSL from the PostgreSQL server. Note that this means you - can't use SCRAM authentication with channel binding. - -timeout - This is the time in seconds before the connection to the server will time out. The - default is ``None`` which means no timeout. - -tcp_keepalive - If ``True`` then use `TCP keepalive - <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``. - -application_name - Sets the `application_name - <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_. - If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default - is ``None`` which means that the server will set the application name. - -replication - Used to run in `streaming replication mode - <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide values as - bytes, eg. ``'database'.encode('EUC-JP')``. - -sock - A socket-like object to use for the connection. For example, ``sock`` could be a plain - ``socket.socket``, or it could represent an SSH tunnel or perhaps an - ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be - used to attempt to create an SSL socket from the provided socket. - -pg8000.native.Connection.notifications -`````````````````````````````````````` - -A deque of server-side `notifications -<https://www.postgresql.org/docs/current/sql-notify.html>`__ received by this database -connection (via the ``LISTEN`` / ``NOTIFY`` PostgreSQL commands). Each list item is a -three-element tuple containing the PostgreSQL backend PID that issued the notify, the -channel and the payload. - - -pg8000.native.Connection.notices -```````````````````````````````` - -A deque of server-side notices received by this database connection. - - -pg8000.native.Connection.parameter_statuses -``````````````````````````````````````````` - -A deque of server-side parameter statuses received by this database connection. - - -pg8000.native.Connection.run(sql, stream=None, types=None, \*\*kwargs) -`````````````````````````````````````````````````````````````````````` - -Executes an sql statement, and returns the results as a ``list``. For example:: - - con.run("SELECT * FROM cities where population > :pop", pop=10000) - -sql - The SQL statement to execute. Parameter placeholders appear as a ``:`` followed by the - parameter name. - -stream - For use with the PostgreSQL `COPY - <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. The nature - of the parameter depends on whether the SQL command is ``COPY FROM`` or ``COPY TO``. - - ``COPY FROM`` - The stream parameter must be a readable file-like object or an iterable. If it's an - iterable then the items can be ``str`` or binary. - ``COPY TO`` - The stream parameter must be a writable file-like object. - -types - A dictionary of oids. A key corresponds to a parameter. - -kwargs - The parameters of the SQL statement. - - -pg8000.native.Connection.row_count -`````````````````````````````````` - -This read-only attribute contains the number of rows that the last ``run()`` method -produced (for query statements like ``SELECT``) or affected (for modification statements -like ``UPDATE``. - -The value is -1 if: - -- No ``run()`` method has been performed yet. -- There was no rowcount associated with the last ``run()``. - - -pg8000.native.Connection.columns -```````````````````````````````` - -A list of column metadata. Each item in the list is a dictionary with the following -keys: - -- name -- table_oid -- column_attrnum -- type_oid -- type_size -- type_modifier -- format - - -pg8000.native.Connection.close() -```````````````````````````````` - -Closes the database connection. - - -pg8000.native.Connection.register_out_adapter(typ, out_func) -```````````````````````````````````````````````````````````` - -Register a type adapter for types going out from pg8000 to the server. - -typ - The Python class that the adapter is for. - -out_func - A function that takes the Python object and returns its string representation - in the format that the server requires. - - -pg8000.native.Connection.register_in_adapter(oid, in_func) -`````````````````````````````````````````````````````````` - -Register a type adapter for types coming in from the server to pg8000. - -oid - The PostgreSQL type identifier found in the `pg_type system catalog - <https://www.postgresql.org/docs/current/catalog-pg-type.html>`_. - -in_func - A function that takes the PostgreSQL string representation and returns a corresponding - Python object. - - -pg8000.native.Connection.prepare(sql) -````````````````````````````````````` - -Returns a ``PreparedStatement`` object which represents a `prepared statement -<https://www.postgresql.org/docs/current/sql-prepare.html>`_ on the server. It can -subsequently be repeatedly executed. - -sql - The SQL statement to prepare. Parameter placeholders appear as a ``:`` followed by the - parameter name. - - -pg8000.native.PreparedStatement -``````````````````````````````` - -A prepared statement object is returned by the ``pg8000.native.Connection.prepare()`` -method of a connection. It has the following methods: - - -pg8000.native.PreparedStatement.run(\*\*kwargs) -``````````````````````````````````````````````` - -Executes the prepared statement, and returns the results as a ``tuple``. - -kwargs - The parameters of the prepared statement. - - -pg8000.native.PreparedStatement.close() -``````````````````````````````````````` - -Closes the prepared statement, releasing the prepared statement held on the server. - - -pg8000.native.identifier(ident) -``````````````````````````````` - -Correctly quotes and escapes a string to be used as an `SQL identifier -<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS>`_. - -ident - The ``str`` to be used as an SQL identifier. - - -pg8000.native.literal(value) -```````````````````````````` - -Correctly quotes and escapes a value to be used as an `SQL literal -<https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS>`_. - -value - The value to be used as an SQL literal. +`Native API Docs <docs/native_api_docs.rst>`_ DB-API 2 Docs ------------- - -Properties -`````````` - - -pg8000.dbapi.apilevel -::::::::::::::::::::: - -The DBAPI level supported, currently "2.0". - - -pg8000.dbapi.threadsafety -::::::::::::::::::::::::: - -Integer constant stating the level of thread safety the DBAPI interface supports. For -pg8000, the threadsafety value is 1, meaning that threads may share the module but not -connections. - - -pg8000.dbapi.paramstyle -::::::::::::::::::::::: - -String property stating the type of parameter marker formatting expected by -the interface. This value defaults to "format", in which parameters are -marked in this format: "WHERE name=%s". - -As an extension to the DBAPI specification, this value is not constant; it can be -changed to any of the following values: - -qmark - Question mark style, eg. ``WHERE name=?`` - -numeric - Numeric positional style, eg. ``WHERE name=:1`` - -named - Named style, eg. ``WHERE name=:paramname`` - -format - printf format codes, eg. ``WHERE name=%s`` - -pyformat - Python format codes, eg. ``WHERE name=%(paramname)s`` - - -pg8000.dbapi.STRING -::::::::::::::::::: - -String type oid. - -pg8000.dbapi.BINARY -::::::::::::::::::: - - -pg8000.dbapi.NUMBER -::::::::::::::::::: - -Numeric type oid. - - -pg8000.dbapi.DATETIME -::::::::::::::::::::: - -Timestamp type oid - - -pg8000.dbapi.ROWID -:::::::::::::::::: - -ROWID type oid - - -Functions -````````` - -pg8000.dbapi.connect(user, host='localhost', database=None, port=5432, password=None, source_address=None, unix_sock=None, ssl_context=None, timeout=None, tcp_keepalive=True, application_name=None, replication=None, sock=None) -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -Creates a connection to a PostgreSQL database. - -user - The username to connect to the PostgreSQL server with. If your server character - encoding is not ``ascii`` or ``utf8``, then you need to provide ``user`` as bytes, - eg. ``'my_name'.encode('EUC-JP')``. - -host - The hostname of the PostgreSQL server to connect with. Providing this parameter is - necessary for TCP/IP connections. One of either ``host`` or ``unix_sock`` must be - provided. The default is ``localhost``. - -database - The name of the database instance to connect with. If ``None`` then the PostgreSQL - server will assume the database name is the same as the username. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide ``database`` - as bytes, eg. ``'my_db'.encode('EUC-JP')``. - -port - The TCP/IP port of the PostgreSQL server instance. This parameter defaults to - ``5432``, the registered common port of PostgreSQL TCP/IP servers. - -password - The user password to connect to the server with. This parameter is optional; if - omitted and the database server requests password-based authentication, the - connection will fail to open. If this parameter is provided but not requested by the - server, no error will occur. - - If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide ``password`` as bytes, eg. ``'my_password'.encode('EUC-JP')``. - -source_address - The source IP address which initiates the connection to the PostgreSQL server. The - default is ``None`` which means that the operating system will choose the source - address. - -unix_sock - The path to the UNIX socket to access the database through, for example, - ``'/tmp/.s.PGSQL.5432'``. One of either ``host`` or ``unix_sock`` must be provided. - -ssl_context - This governs SSL encryption for TCP/IP sockets. It can have three values: - - - ``None``, meaning no SSL (the default) - - ``True``, means use SSL with an |ssl.SSLContext|_ created using - |ssl.create_default_context()|_. - - - An instance of |ssl.SSLContext|_ which will be used to create the SSL connection. - - If your PostgreSQL server is behind an SSL proxy, you can set the pg8000-specific - attribute ``ssl.SSLContext.request_ssl = False``, which tells pg8000 to use an SSL - socket, but not to request SSL from the PostgreSQL server. Note that this means you - can't use SCRAM authentication with channel binding. - -timeout - This is the time in seconds before the connection to the server will time out. The - default is ``None`` which means no timeout. - -tcp_keepalive - If ``True`` then use `TCP keepalive - <https://en.wikipedia.org/wiki/Keepalive#TCP_keepalive>`_. The default is ``True``. - -application_name - Sets the `application_name - <https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-APPLICATION-NAME>`_. If your server character encoding is not ``ascii`` or ``utf8``, then you need to - provide values as bytes, eg. ``'my_application_name'.encode('EUC-JP')``. The default - is ``None`` which means that the server will set the application name. - -replication - Used to run in `streaming replication mode - <https://www.postgresql.org/docs/current/protocol-replication.html>`_. If your server - character encoding is not ``ascii`` or ``utf8``, then you need to provide values as - bytes, eg. ``'database'.encode('EUC-JP')``. - -sock - A socket-like object to use for the connection. For example, ``sock`` could be a plain - ``socket.socket``, or it could represent an SSH tunnel or perhaps an - ``ssl.SSLSocket`` to an SSL proxy. If an |ssl.SSLContext| is provided, then it will be - used to attempt to create an SSL socket from the provided socket. - - -pg8000.dbapi.Date(year, month, day) - -Construct an object holding a date value. - -This property is part of the `DBAPI 2.0 specification -<http://www.python.org/dev/peps/pep-0249/>`_. - -Returns: `datetime.date` - - -pg8000.dbapi.Time(hour, minute, second) -::::::::::::::::::::::::::::::::::::::: - -Construct an object holding a time value. - -Returns: ``datetime.time`` - - -pg8000.dbapi.Timestamp(year, month, day, hour, minute, second) -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -Construct an object holding a timestamp value. - -Returns: ``datetime.datetime`` - - -pg8000.dbapi.DateFromTicks(ticks) -::::::::::::::::::::::::::::::::: - -Construct an object holding a date value from the given ticks value (number of seconds -since the epoch). - -Returns: ``datetime.datetime`` - - -pg8000.dbapi.TimeFromTicks(ticks) -::::::::::::::::::::::::::::::::: - -Construct an object holding a time value from the given ticks value (number of seconds -since the epoch). - -Returns: ``datetime.time`` - - -pg8000.dbapi.TimestampFromTicks(ticks) -:::::::::::::::::::::::::::::::::::::: - -Construct an object holding a timestamp value from the given ticks value (number of -seconds since the epoch). - -Returns: ``datetime.datetime`` - - -pg8000.dbapi.Binary(value) -:::::::::::::::::::::::::: - -Construct an object holding binary data. - -Returns: ``bytes``. - - -Generic Exceptions -`````````````````` - -Pg8000 uses the standard DBAPI 2.0 exception tree as "generic" exceptions. Generally, -more specific exception types are raised; these specific exception types are derived -from the generic exceptions. - -pg8000.dbapi.Warning -:::::::::::::::::::: - -Generic exception raised for important database warnings like data truncations. This -exception is not currently used by pg8000. - - -pg8000.dbapi.Error -:::::::::::::::::: - -Generic exception that is the base exception of all other error exceptions. - - -pg8000.dbapi.InterfaceError -::::::::::::::::::::::::::: - -Generic exception raised for errors that are related to the database interface rather -than the database itself. For example, if the interface attempts to use an SSL -connection but the server refuses, an InterfaceError will be raised. - - -pg8000.dbapi.DatabaseError -:::::::::::::::::::::::::: - -Generic exception raised for errors that are related to the database. This exception is -currently never raised by pg8000. - - -pg8000.dbapi.DataError -:::::::::::::::::::::: - -Generic exception raised for errors that are due to problems with the processed data. -This exception is not currently raised by pg8000. - - -pg8000.dbapi.OperationalError -::::::::::::::::::::::::::::: - -Generic exception raised for errors that are related to the database's operation and not -necessarily under the control of the programmer. This exception is currently never -raised by pg8000. - - -pg8000.dbapi.IntegrityError -::::::::::::::::::::::::::: - -Generic exception raised when the relational integrity of the database is affected. This -exception is not currently raised by pg8000. - - -pg8000.dbapi.InternalError -:::::::::::::::::::::::::: - -Generic exception raised when the database encounters an internal error. This is -currently only raised when unexpected state occurs in the pg8000 interface itself, and -is typically the result of a interface bug. - - -pg8000.dbapi.ProgrammingError -::::::::::::::::::::::::::::: - -Generic exception raised for programming errors. For example, this exception is raised -if more parameter fields are in a query string than there are available parameters. - - -pg8000.dbapi.NotSupportedError -:::::::::::::::::::::::::::::: - -Generic exception raised in case a method or database API was used which is not -supported by the database. - - -Classes -``````` - - -pg8000.dbapi.Connection -::::::::::::::::::::::: - -A connection object is returned by the ``pg8000.connect()`` function. It represents a -single physical connection to a PostgreSQL database. - - -pg8000.dbapi.Connection.autocommit -:::::::::::::::::::::::::::::::::: - -Following the DB-API specification, autocommit is off by default. It can be turned on by -setting this boolean pg8000-specific autocommit property to ``True``. - - -pg8000.dbapi.Connection.close() -::::::::::::::::::::::::::::::: - -Closes the database connection. - - -pg8000.dbapi.Connection.cursor() -:::::::::::::::::::::::::::::::: - -Creates a ``pg8000.dbapi.Cursor`` object bound to this connection. - - -pg8000.dbapi.Connection.rollback() -:::::::::::::::::::::::::::::::::: - -Rolls back the current database transaction. - - -pg8000.dbapi.Connection.tpc_begin(xid) -:::::::::::::::::::::::::::::::::::::: - -Begins a TPC transaction with the given transaction ID xid. This method should be -called outside of a transaction (i.e. nothing may have executed since the last -``commit()`` or ``rollback()``. Furthermore, it is an error to call ``commit()`` or -``rollback()`` within the TPC transaction. A ``ProgrammingError`` is raised, if the -application calls ``commit()`` or ``rollback()`` during an active TPC transaction. - - -pg8000.dbapi.Connection.tpc_commit(xid=None) -:::::::::::::::::::::::::::::::::::::::::::: - -When called with no arguments, ``tpc_commit()`` commits a TPC transaction previously -prepared with ``tpc_prepare()``. If ``tpc_commit()`` is called prior to -``tpc_prepare()``, a single phase commit is performed. A transaction manager may choose -to do this if only a single resource is participating in the global transaction. - -When called with a transaction ID ``xid``, the database commits the given transaction. -If an invalid transaction ID is provided, a ``ProgrammingError`` will be raised. This -form should be called outside of a transaction, and is intended for use in recovery. - -On return, the TPC transaction is ended. - - -pg8000.dbapi.Connection.tpc_prepare() -::::::::::::::::::::::::::::::::::::: - -Performs the first phase of a transaction started with ``.tpc_begin()``. A -``ProgrammingError`` is be raised if this method is called outside of a TPC transaction. - -After calling ``tpc_prepare()``, no statements can be executed until ``tpc_commit()`` or -``tpc_rollback()`` have been called. - - -pg8000.dbapi.Connection.tpc_recover() -::::::::::::::::::::::::::::::::::::: - -Returns a list of pending transaction IDs suitable for use with ``tpc_commit(xid)`` or -``tpc_rollback(xid)``. - - -pg8000.dbapi.Connection.tpc_rollback(xid=None) -:::::::::::::::::::::::::::::::::::::::::::::: - -When called with no arguments, ``tpc_rollback()`` rolls back a TPC transaction. It may -be called before or after ``tpc_prepare()``. - -When called with a transaction ID xid, it rolls back the given transaction. If an -invalid transaction ID is provided, a ``ProgrammingError`` is raised. This form should -be called outside of a transaction, and is intended for use in recovery. - -On return, the TPC transaction is ended. - - -pg8000.dbapi.Connection.xid(format_id, global_transaction_id, branch_qualifier) -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - -Create a Transaction IDs (only global_transaction_id is used in pg) format_id and -branch_qualifier are not used in postgres global_transaction_id may be any string -identifier supported by postgres returns a tuple (format_id, global_transaction_id, -branch_qualifier) - - -pg8000.dbapi.Cursor -::::::::::::::::::: - -A cursor object is returned by the ``pg8000.dbapi.Connection.cursor()`` method of a -connection. It has the following attributes and methods: - -pg8000.dbapi.Cursor.arraysize -''''''''''''''''''''''''''''' - -This read/write attribute specifies the number of rows to fetch at a time with -``pg8000.dbapi.Cursor.fetchmany()``. It defaults to 1. - - -pg8000.dbapi.Cursor.connection -'''''''''''''''''''''''''''''' - -This read-only attribute contains a reference to the connection object (an instance of -``pg8000.dbapi.Connection``) on which the cursor was created. - - -pg8000.dbapi.Cursor.rowcount -'''''''''''''''''''''''''''' - -This read-only attribute contains the number of rows that the last ``execute()`` or -``executemany()`` method produced (for query statements like ``SELECT``) or affected -(for modification statements like ``UPDATE``. - -The value is -1 if: - -- No ``execute()`` or ``executemany()`` method has been performed yet on the cursor. - -- There was no rowcount associated with the last ``execute()``. - -- At least one of the statements executed as part of an ``executemany()`` had no row - count associated with it. - - -pg8000.dbapi.Cursor.description -''''''''''''''''''''''''''''''' - -This read-only attribute is a sequence of 7-item sequences. Each value contains -information describing one result column. The 7 items returned for each column are -(name, type_code, display_size, internal_size, precision, scale, null_ok). Only the -first two values are provided by the current implementation. - - -pg8000.dbapi.Cursor.close() -''''''''''''''''''''''''''' - -Closes the cursor. - - -pg8000.dbapi.Cursor.execute(operation, args=None, stream=None) -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Executes a database operation. Parameters may be provided as a sequence, or as a -mapping, depending upon the value of ``pg8000.dbapi.paramstyle``. Returns the cursor, -which may be iterated over. - -operation - The SQL statement to execute. - -args - If ``pg8000.dbapi.paramstyle`` is ``qmark``, ``numeric``, or ``format``, this - argument should be an array of parameters to bind into the statement. If - ``pg8000.dbapi.paramstyle`` is ``named``, the argument should be a ``dict`` mapping of - parameters. If ``pg8000.dbapi.paramstyle`` is ``pyformat``, the argument value may be - either an array or a mapping. - -stream - This is a pg8000 extension for use with the PostgreSQL `COPY - <http://www.postgresql.org/docs/current/static/sql-copy.html>`__ command. For a - ``COPY FROM`` the parameter must be a readable file-like object, and for ``COPY TO`` - it must be writable. - - -pg8000.dbapi.Cursor.executemany(operation, param_sets) -'''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Prepare a database operation, and then execute it against all parameter sequences or -mappings provided. - -operation - The SQL statement to execute. - -parameter_sets - A sequence of parameters to execute the statement with. The values in the sequence - should be sequences or mappings of parameters, the same as the args argument of the - ``pg8000.dbapi.Cursor.execute()`` method. - - -pg8000.dbapi.Cursor.callproc(procname, parameters=None) -''''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Call a stored database procedure with the given name and optional parameters. - - -procname - The name of the procedure to call. - -parameters - A list of parameters. - - -pg8000.dbapi.Cursor.fetchall() -'''''''''''''''''''''''''''''' - -Fetches all remaining rows of a query result. - -Returns: A sequence, each entry of which is a sequence of field values making up a row. - - -pg8000.dbapi.Cursor.fetchmany(size=None) -'''''''''''''''''''''''''''''''''''''''' - -Fetches the next set of rows of a query result. - -size - The number of rows to fetch when called. If not provided, the - ``pg8000.dbapi.Cursor.arraysize`` attribute value is used instead. - -Returns: A sequence, each entry of which is a sequence of field values making up a row. -If no more rows are available, an empty sequence will be returned. - - -pg8000.dbapi.Cursor.fetchone() -'''''''''''''''''''''''''''''' - -Fetch the next row of a query result set. - -Returns: A row as a sequence of field values, or ``None`` if no more rows are available. - - -pg8000.dbapi.Cursor.setinputsizes(\*sizes) -'''''''''''''''''''''''''''''''''''''''''' - -Used to set the parameter types of the next query. This is useful if it's difficult for -pg8000 to work out the types from the parameters themselves (eg. for parameters of type -None). - -sizes - Positional parameters that are either the Python type of the parameter to be sent, or - the PostgreSQL oid. Common oids are available as constants such as ``pg8000.STRING``, - ``pg8000.INTEGER``, ``pg8000.TIME`` etc. - - -pg8000.dbapi.Cursor.setoutputsize(size, column=None) -'''''''''''''''''''''''''''''''''''''''''''''''''''' - -Not implemented by pg8000. - - -pg8000.dbapi.Interval -''''''''''''''''''''' - -An Interval represents a measurement of time. In PostgreSQL, an interval is defined in -the measure of months, days, and microseconds; as such, the pg8000 interval type -represents the same information. - -Note that values of the ``pg8000.dbapi.Interval.microseconds``, -``pg8000.dbapi.Interval.days``, and ``pg8000.dbapi.Interval.months`` properties are -independently measured and cannot be converted to each other. A month may be 28, 29, 30, -or 31 days, and a day may occasionally be lengthened slightly by a leap second. +`DB-API 2 Docs <docs/dbapi2_docs.rst>`_ Design Decisions @@ -2230,243 +1404,4 @@ Run ``tox`` to make sure all tests pass, then update the release notes, then do: Release Notes ------------- -Version 1.30.4, 2024-01-03 -`````````````````````````` - -- Add support for more range and multirange types. - -- Make the ``Connection.parameter_statuses`` property a ``dict`` rather than a ``dequeue``. - - -Version 1.30.3, 2023-10-31 -`````````````````````````` - -- Fix problem with PG date overflowing Python types. Now we return the ``str`` we got from the - server if we can't parse it. - - -Version 1.30.2, 2023-09-17 -`````````````````````````` - -- Bug fix where dollar-quoted string constants weren't supported. - - -Version 1.30.1, 2023-07-29 -`````````````````````````` - -- There was a problem uploading the previous version (1.30.0) to PyPI because the - markup of the README.rst was invalid. There's now a step in the automated tests to - check for this. - - -Version 1.30.0, 2023-07-27 -`````````````````````````` - -- Remove support for Python 3.7 - -- Add a ``sock`` keyword parameter for creating a connection from a pre-configured - socket. - - -Version 1.29.8, 2023-06-16 -`````````````````````````` - -- Ranges don't work with legacy API. - - -Version 1.29.7, 2023-06-16 -`````````````````````````` - -- Add support for PostgreSQL ``range`` and ``multirange`` types. Previously pg8000 - would just return them as strings, but now they're returned as ``Range`` and lists of - ``Range``. - -- The PostgreSQL ``record`` type is now returned as a ``tuple`` of strings, whereas - before it was returned as one string. - - -Version 1.29.6, 2023-05-29 -`````````````````````````` - -- Fixed two bugs with composite types. Nulls should be represented by an empty string, - and in an array of composite types, the elements should be surrounded by double - quotes. - - -Version 1.29.5, 2023-05-09 -`````````````````````````` - -- Fixed bug where pg8000 didn't handle the case when the number of bytes received from - a socket was fewer than requested. This was being interpreted as a network error, but - in fact we just needed to wait until more bytes were available. - -- When using the ``PGInterval`` type, if a response from the server contained the period - ``millennium``, it wasn't recognised. This was caused by a spelling mistake where we - had ``millenium`` rather than ``millennium``. - -- Added support for sending PostgreSQL composite types. If a value is sent as a - ``tuple``, pg8000 will send it to the server as a ``(`` delimited composite string. - - -Version 1.29.4, 2022-12-14 -`````````````````````````` - -- Fixed bug in ``pg8000.dbapi`` in the ``setinputsizes()`` method where if a ``size`` - was a recognized Python type, the method failed. - - -Version 1.29.3, 2022-10-26 -`````````````````````````` - -- Upgrade the SCRAM library to version 1.4.3. This adds support for the case where the - client supports channel binding but the server doesn't. - - -Version 1.29.2, 2022-10-09 -`````````````````````````` - -- Fixed a bug where in a literal array, items such as ``\n`` and ``\r`` weren't - escaped properly before being sent to the server. - -- Fixed a bug where if the PostgreSQL server has a half-hour time zone set, values of - type ``timestamp with time zone`` failed. This has been fixed by using the ``parse`` - function of the ``dateutil`` package if the ``datetime`` parser fails. - - -Version 1.29.1, 2022-05-23 -`````````````````````````` - -- In trying to determine if there's been a failed commit, check for ``ROLLBACK TO - SAVEPOINT``. - - -Version 1.29.0, 2022-05-21 -`````````````````````````` - -- Implement a workaround for the `silent failed commit - <https://github.com/tlocke/pg8000/issues/36>`_ bug. - -- Previously if an empty string was sent as the query an exception would be raised, but - that isn't done now. - - -Version 1.28.3, 2022-05-18 -`````````````````````````` - -- Put back ``__version__`` attributes that were inadvertently removed. - - -Version 1.28.2, 2022-05-17 -`````````````````````````` - -- Use a build system that's compliant with PEP517. - - -Version 1.28.1, 2022-05-17 -`````````````````````````` - -- If when doing a ``COPY FROM`` the ``stream`` parameter is an iterator of ``str``, - pg8000 used to silently append a newline to the end. That no longer happens. - - -Version 1.28.0, 2022-05-17 -`````````````````````````` - -- When using the ``COPY FROM`` SQL statement, allow the ``stream`` parameter to be an - iterable. - - -Version 1.27.1, 2022-05-16 -`````````````````````````` - -- The ``seconds`` attribute of ``PGInterval`` is now always a ``float``, to cope with - fractional seconds. - -- Updated the ``interval`` parsers for ``iso_8601`` and ``sql_standard`` to take - account of fractional seconds. - - -Version 1.27.0, 2022-05-16 -`````````````````````````` - -- It used to be that by default, if pg8000 received an ``interval`` type from the server - and it was too big to fit into a ``datetime.timedelta`` then an exception would be - raised. Now if an interval is too big for ``datetime.timedelta`` a ``PGInterval`` is - returned. - -* pg8000 now supports all the output formats for an ``interval`` (``postgres``, - ``postgres_verbose``, ``iso_8601`` and ``sql_standard``). - - -Version 1.26.1, 2022-04-23 -`````````````````````````` - -- Make sure all tests are run by the GitHub Actions tests on commit. -- Remove support for Python 3.6 -- Remove support for PostgreSQL 9.6 - - -Version 1.26.0, 2022-04-18 -`````````````````````````` - -- When connecting, raise an ``InterfaceError('network error')`` rather than let the - underlying ``struct.error`` float up. - -- Make licence text the same as that used by the OSI. Previously the licence wording - differed slightly from the BSD 3 Clause licence at - https://opensource.org/licenses/BSD-3-Clause. This meant that automated tools didn't - pick it up as being Open Source. The changes are believed to not alter the meaning of the license at all. - - -Version 1.25.0, 2022-04-17 -`````````````````````````` - -- Fix more cases where a ``ResourceWarning`` would be raise because of a socket that had - been left open. - -- We now have a single ``InterfaceError`` with the message 'network error' for all - network errors, with the underlying exception held in the ``cause`` of the exception. - - -Version 1.24.2, 2022-04-15 -`````````````````````````` - -- To prevent a ``ResourceWarning`` close socket if a connection can't be created. - - -Version 1.24.1, 2022-03-02 -`````````````````````````` - -- Return pg +/-infinity dates as ``str``. Previously +/-infinity pg values would cause - an error when returned, but now we return +/-infinity as strings. - - -Version 1.24.0, 2022-02-06 -`````````````````````````` - -- Add SQL escape functions identifier() and literal() to the native API. For use when a - query can't be parameterised and the SQL string has to be created using untrusted - values. - - -Version 1.23.0, 2021-11-13 -`````````````````````````` - -- If a query has no parameters, then the query will no longer be parsed. Although there - are performance benefits for doing this, the main reason is to avoid query rewriting, - which can introduce errors. - - -Version 1.22.1, 2021-11-10 -`````````````````````````` - -- Fix bug in PGInterval type where ``str()`` failed for a millennia value. - - -Version 1.22.0, 2021-10-13 -`````````````````````````` - -- Rather than specifying the oids in the ``Parse`` step of the Postgres protocol, pg8000 - now omits them, and so Postgres will use the oids it determines from the query. This - makes the pg8000 code simpler and also it should also make the nuances of type - matching more straightforward. +`Release Notes <docs/release_notes.rst>`_ diff --git a/contrib/python/pg8000/pg8000/core.py b/contrib/python/pg8000/pg8000/core.py index 3e55957755..8375b95213 100644 --- a/contrib/python/pg8000/pg8000/core.py +++ b/contrib/python/pg8000/pg8000/core.py @@ -28,7 +28,7 @@ def pack_funcs(fmt): i_pack, i_unpack = pack_funcs("i") -h_pack, h_unpack = pack_funcs("h") +H_pack, H_unpack = pack_funcs("H") ii_pack, ii_unpack = pack_funcs("ii") ihihih_pack, ihihih_unpack = pack_funcs("ihihih") ci_pack, ci_unpack = pack_funcs("ci") @@ -623,7 +623,7 @@ class CoreConnection: self._backend_key_data = data def handle_ROW_DESCRIPTION(self, data, context): - count = h_unpack(data)[0] + count = H_unpack(data)[0] idx = 2 columns = [] input_funcs = [] @@ -656,7 +656,7 @@ class CoreConnection: def send_PARSE(self, statement_name_bin, statement, oids=()): val = bytearray(statement_name_bin) val.extend(statement.encode(self._client_encoding) + NULL_BYTE) - val.extend(h_pack(len(oids))) + val.extend(H_pack(len(oids))) for oid in oids: val.extend(i_pack(0 if oid == -1 else oid)) @@ -762,7 +762,7 @@ class CoreConnection: """https://www.postgresql.org/docs/current/protocol-message-formats.html""" retval = bytearray( - NULL_BYTE + statement_name_bin + h_pack(0) + h_pack(len(params)) + NULL_BYTE + statement_name_bin + H_pack(0) + H_pack(len(params)) ) for value in params: @@ -772,7 +772,7 @@ class CoreConnection: val = value.encode(self._client_encoding) retval.extend(i_pack(len(val))) retval.extend(val) - retval.extend(h_pack(0)) + retval.extend(H_pack(0)) self._send_message(BIND, retval) _write(self._sock, FLUSH_MSG) diff --git a/contrib/python/pg8000/ya.make b/contrib/python/pg8000/ya.make index 5067622704..f04dfe2320 100644 --- a/contrib/python/pg8000/ya.make +++ b/contrib/python/pg8000/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.30.4) +VERSION(1.30.5) LICENSE(BSD-3-Clause) diff --git a/yt/yt/core/misc/optional.h b/library/cpp/yt/misc/optional.h index cfae5c36d5..cfae5c36d5 100644 --- a/yt/yt/core/misc/optional.h +++ b/library/cpp/yt/misc/optional.h diff --git a/yt/yt/core/concurrency/fair_share_action_queue-inl.h b/yt/yt/core/concurrency/fair_share_action_queue-inl.h index 11cf50e4eb..5c3f95dc0e 100644 --- a/yt/yt/core/concurrency/fair_share_action_queue-inl.h +++ b/yt/yt/core/concurrency/fair_share_action_queue-inl.h @@ -16,8 +16,9 @@ public: TEnumIndexedFairShareActionQueue( const TString& threadName, const std::vector<TString>& queueNames, - const THashMap<TString, std::vector<TString>>& queueToBucket) - : Queue_(CreateFairShareActionQueue(threadName, queueNames, queueToBucket)) + const THashMap<TString, std::vector<TString>>& bucketToQueues, + NProfiling::IRegistryImplPtr registry) + : Queue_(CreateFairShareActionQueue(threadName, queueNames, bucketToQueues, std::move(registry))) { } const IInvokerPtr& GetInvoker(EQueue queue) override @@ -39,21 +40,22 @@ private: template <typename EQueue, typename EBucket> IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueue( const TString& threadName, - const THashMap<EBucket, std::vector<EQueue>>& queueToBucket) + const THashMap<EBucket, std::vector<EQueue>>& bucketToQueues, + NProfiling::IRegistryImplPtr registry) { std::vector<TString> queueNames; for (const auto& queueName : TEnumTraits<EQueue>::GetDomainNames()) { queueNames.push_back(TString{queueName}); } THashMap<TString, std::vector<TString>> stringBuckets; - for (const auto& [bucketName, bucket] : queueToBucket) { + for (const auto& [bucketName, bucket] : bucketToQueues) { auto& stringBucket = stringBuckets[ToString(bucketName)]; stringBucket.reserve(bucket.size()); for (const auto& queue : bucket) { stringBucket.push_back(ToString(queue)); } } - return New<TEnumIndexedFairShareActionQueue<EQueue>>(threadName, queueNames, stringBuckets); + return New<TEnumIndexedFairShareActionQueue<EQueue>>(threadName, queueNames, stringBuckets, std::move(registry)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fair_share_action_queue.cpp b/yt/yt/core/concurrency/fair_share_action_queue.cpp index 21357112cd..dae24f21ba 100644 --- a/yt/yt/core/concurrency/fair_share_action_queue.cpp +++ b/yt/yt/core/concurrency/fair_share_action_queue.cpp @@ -29,7 +29,8 @@ public: TFairShareActionQueue( const TString& threadName, const std::vector<TString>& queueNames, - const THashMap<TString, std::vector<TString>>& queueToBucket) + const THashMap<TString, std::vector<TString>>& bucketToQueues, + NProfiling::IRegistryImplPtr registry) : ShutdownCookie_(RegisterShutdownCallback( Format("FairShareActionQueue(%v)", threadName), BIND_NO_PROPAGATE(&TFairShareActionQueue::Shutdown, MakeWeak(this), /*graceful*/ false), @@ -37,7 +38,7 @@ public: { THashMap<TString, int> queueNameToIndex; for (int queueIndex = 0; queueIndex < std::ssize(queueNames); ++queueIndex) { - YT_VERIFY(queueNameToIndex.emplace(queueNames[queueIndex], queueIndex).second); + EmplaceOrCrash(queueNameToIndex, queueNames[queueIndex], queueIndex); } QueueIndexToBucketIndex_.resize(queueNames.size(), -1); @@ -47,18 +48,20 @@ public: std::vector<TBucketDescription> bucketDescriptions; THashSet<TString> createdQueues; int nextBucketIndex = 0; - for (const auto& [bucketName, bucketQueues] : queueToBucket) { + for (const auto& [bucketName, bucketQueues] : bucketToQueues) { int bucketIndex = nextBucketIndex++; auto& bucketDescription = bucketDescriptions.emplace_back(); - bucketDescription.BucketTagSet = GetBucketTags(threadName, bucketName); for (int bucketQueueIndex = 0; bucketQueueIndex < std::ssize(bucketQueues); ++bucketQueueIndex) { const auto& queueName = bucketQueues[bucketQueueIndex]; - auto queueIndex = GetOrCrash(queueNameToIndex, queueName); - YT_VERIFY(createdQueues.insert(queueName).second); - QueueIndexToBucketIndex_[queueIndex] = bucketIndex; - QueueIndexToBucketQueueIndex_[queueIndex] = bucketQueueIndex; - bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, queueName)); + bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, bucketName, queueName)); bucketDescription.QueueProfilerTags.push_back(New<NYTProf::TProfilerTag>("queue", queueName)); + + { + int queueIndex = GetOrCrash(queueNameToIndex, queueName); + InsertOrCrash(createdQueues, queueName); + QueueIndexToBucketIndex_[queueIndex] = bucketIndex; + QueueIndexToBucketQueueIndex_[queueIndex] = bucketQueueIndex; + } } BucketNames_.push_back(bucketName); } @@ -70,13 +73,16 @@ public: } auto& bucketDescription = bucketDescriptions.emplace_back(); - bucketDescription.BucketTagSet = GetBucketTags(threadName, queueName); - bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, queueName)); + bucketDescription.QueueTagSets.push_back(GetQueueTags(threadName, queueName, queueName)); bucketDescription.QueueProfilerTags.push_back(New<NYTProf::TProfilerTag>("queue", queueName)); - auto queueIndex = GetOrCrash(queueNameToIndex, queueName); - QueueIndexToBucketIndex_[queueIndex] = nextBucketIndex++; - QueueIndexToBucketQueueIndex_[queueIndex] = 0; - YT_VERIFY(createdQueues.emplace(queueName).second); + + { + auto queueIndex = GetOrCrash(queueNameToIndex, queueName); + QueueIndexToBucketIndex_[queueIndex] = nextBucketIndex++; + QueueIndexToBucketQueueIndex_[queueIndex] = 0; + InsertOrCrash(createdQueues, queueName); + } + BucketNames_.push_back(queueName); } @@ -88,7 +94,7 @@ public: YT_VERIFY(QueueIndexToBucketQueueIndex_[queueIndex] != -1); } - Queue_ = New<TFairShareInvokerQueue>(CallbackEventCount_, std::move(bucketDescriptions)); + Queue_ = New<TFairShareInvokerQueue>(CallbackEventCount_, std::move(bucketDescriptions), std::move(registry)); Thread_ = New<TFairShareQueueSchedulerThread>(Queue_, CallbackEventCount_, threadName, threadName); } @@ -171,9 +177,10 @@ private: IFairShareActionQueuePtr CreateFairShareActionQueue( const TString& threadName, const std::vector<TString>& queueNames, - const THashMap<TString, std::vector<TString>>& queueToBucket) + const THashMap<TString, std::vector<TString>>& bucketToQueues, + NProfiling::IRegistryImplPtr registry) { - return New<TFairShareActionQueue>(threadName, queueNames, queueToBucket); + return New<TFairShareActionQueue>(threadName, queueNames, bucketToQueues, std::move(registry)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fair_share_action_queue.h b/yt/yt/core/concurrency/fair_share_action_queue.h index f0d69caa50..33ac86a5ad 100644 --- a/yt/yt/core/concurrency/fair_share_action_queue.h +++ b/yt/yt/core/concurrency/fair_share_action_queue.h @@ -6,6 +6,8 @@ #include <yt/yt/core/misc/range.h> +#include <yt/yt/core/profiling/public.h> + namespace NYT::NConcurrency { //////////////////////////////////////////////////////////////////////////////// @@ -25,7 +27,8 @@ DEFINE_REFCOUNTED_TYPE(IFairShareActionQueue) IFairShareActionQueuePtr CreateFairShareActionQueue( const TString& threadName, const std::vector<TString>& queueNames, - const THashMap<TString, std::vector<TString>>& queueToBucket = {}); + const THashMap<TString, std::vector<TString>>& bucketToQueues = {}, + NProfiling::IRegistryImplPtr registry = {}); //////////////////////////////////////////////////////////////////////////////// @@ -43,7 +46,8 @@ struct IEnumIndexedFairShareActionQueue template <typename EQueue, typename EBucket = EQueue> IEnumIndexedFairShareActionQueuePtr<EQueue> CreateEnumIndexedFairShareActionQueue( const TString& threadName, - const THashMap<EBucket, std::vector<EQueue>>& queueToBucket = {}); + const THashMap<EBucket, std::vector<EQueue>>& bucketToQueues = {}, + NProfiling::IRegistryImplPtr registry = {}); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fair_share_invoker_queue.cpp b/yt/yt/core/concurrency/fair_share_invoker_queue.cpp index 099cc8aa52..e42cdc16a6 100644 --- a/yt/yt/core/concurrency/fair_share_invoker_queue.cpp +++ b/yt/yt/core/concurrency/fair_share_invoker_queue.cpp @@ -16,7 +16,8 @@ using namespace NYTree; TFairShareInvokerQueue::TFairShareInvokerQueue( TIntrusivePtr<NThreading::TEventCount> callbackEventCount, - const std::vector<TBucketDescription>& bucketDescriptions) + const std::vector<TBucketDescription>& bucketDescriptions, + NProfiling::IRegistryImplPtr registry) : Weights_(bucketDescriptions.size(), 1.0) { Buckets_.reserve(bucketDescriptions.size()); @@ -26,7 +27,7 @@ TFairShareInvokerQueue::TFairShareInvokerQueue( callbackEventCount, bucketDescription.QueueTagSets, bucketDescription.QueueProfilerTags, - bucketDescription.BucketTagSet); + registry); int profilingTagCount = bucketDescription.QueueTagSets.size(); bucket.Invokers.reserve(profilingTagCount); for (int index = 0; index < profilingTagCount; ++index) { diff --git a/yt/yt/core/concurrency/fair_share_invoker_queue.h b/yt/yt/core/concurrency/fair_share_invoker_queue.h index b625c43bd6..c474ffe104 100644 --- a/yt/yt/core/concurrency/fair_share_invoker_queue.h +++ b/yt/yt/core/concurrency/fair_share_invoker_queue.h @@ -21,7 +21,6 @@ struct TBucketDescription { std::vector<NProfiling::TTagSet> QueueTagSets; std::vector<NYTProf::TProfilerTagPtr> QueueProfilerTags; - NProfiling::TTagSet BucketTagSet; }; //////////////////////////////////////////////////////////////////////////////// @@ -32,7 +31,8 @@ class TFairShareInvokerQueue public: TFairShareInvokerQueue( TIntrusivePtr<NThreading::TEventCount> callbackEventCount, - const std::vector<TBucketDescription>& bucketDescriptions); + const std::vector<TBucketDescription>& bucketDescriptions, + NProfiling::IRegistryImplPtr registry = {}); ~TFairShareInvokerQueue(); diff --git a/yt/yt/core/concurrency/invoker_queue.cpp b/yt/yt/core/concurrency/invoker_queue.cpp index 5c0607e6dd..fec32d6c86 100644 --- a/yt/yt/core/concurrency/invoker_queue.cpp +++ b/yt/yt/core/concurrency/invoker_queue.cpp @@ -287,10 +287,11 @@ private: template <class TQueueImpl> TInvokerQueue<TQueueImpl>::TInvokerQueue( TIntrusivePtr<NThreading::TEventCount> callbackEventCount, - const TTagSet& counterTagSet) + const TTagSet& counterTagSet, + NProfiling::IRegistryImplPtr registry) : CallbackEventCount_(std::move(callbackEventCount)) { - Counters_.push_back(CreateCounters(counterTagSet)); + Counters_.push_back(CreateCounters(counterTagSet, std::move(registry))); } template <class TQueueImpl> @@ -298,18 +299,16 @@ TInvokerQueue<TQueueImpl>::TInvokerQueue( TIntrusivePtr<NThreading::TEventCount> callbackEventCount, const std::vector<TTagSet>& counterTagSets, const std::vector<NYTProf::TProfilerTagPtr>& profilerTags, - const TTagSet& cumulativeCounterTagSet) + NProfiling::IRegistryImplPtr registry) : CallbackEventCount_(std::move(callbackEventCount)) { YT_VERIFY(counterTagSets.size() == profilerTags.size()); Counters_.reserve(counterTagSets.size()); for (const auto& tagSet : counterTagSets) { - Counters_.push_back(CreateCounters(tagSet)); + Counters_.push_back(CreateCounters(tagSet, registry)); } - CumulativeCounters_ = CreateCounters(cumulativeCounterTagSet); - ProfilingTagSettingInvokers_.reserve(Counters_.size()); for (int index = 0; index < std::ssize(Counters_); ++index) { ProfilingTagSettingInvokers_.push_back( @@ -388,14 +387,10 @@ TCpuInstant TInvokerQueue<TQueueImpl>::EnqueueCallback( auto action = MakeAction(std::move(callback), profilingTag, std::move(profilerTag), cpuInstant); - auto updateCounters = [&] (const TCountersPtr& counters) { - if (counters) { - counters->ActiveCallbacks += 1; - counters->EnqueuedCounter.Increment(); - } - }; - updateCounters(Counters_[profilingTag]); - updateCounters(CumulativeCounters_); + if (Counters_[profilingTag]) { + Counters_[profilingTag]->ActiveCallbacks += 1; + Counters_[profilingTag]->EnqueuedCounter.Increment(); + } QueueImpl_.Enqueue(std::move(action)); return cpuInstant; @@ -421,14 +416,10 @@ TCpuInstant TInvokerQueue<TQueueImpl>::EnqueueCallbacks( actions.push_back(MakeAction(std::move(callback), profilingTag, profilerTag, cpuInstant)); } - auto updateCounters = [&] (const TCountersPtr& counters) { - if (counters) { - counters->ActiveCallbacks += std::ssize(actions); - counters->EnqueuedCounter.Increment(std::ssize(actions)); - } - }; - updateCounters(Counters_[profilingTag]); - updateCounters(CumulativeCounters_); + if (Counters_[profilingTag]) { + Counters_[profilingTag]->ActiveCallbacks += std::ssize(actions); + Counters_[profilingTag]->EnqueuedCounter.Increment(std::ssize(actions)); + } QueueImpl_.Enqueue(actions); return cpuInstant; @@ -493,14 +484,10 @@ bool TInvokerQueue<TQueueImpl>::BeginExecute(TEnqueuedAction* action, typename T WaitTimeObserver_(waitTime); } - auto updateCounters = [&] (const TCountersPtr& counters) { - if (counters) { - counters->DequeuedCounter.Increment(); - counters->WaitTimer.Record(waitTime); - } - }; - updateCounters(Counters_[action->ProfilingTag]); - updateCounters(CumulativeCounters_); + if (Counters_[action->ProfilingTag]) { + Counters_[action->ProfilingTag]->DequeuedCounter.Increment(); + Counters_[action->ProfilingTag]->WaitTimer.Record(waitTime); + } if (const auto& profilerTag = action->ProfilerTag) { GetTlsRef(CpuProfilerTagGuard) = TCpuProfilerTagGuard(profilerTag); @@ -532,16 +519,12 @@ void TInvokerQueue<TQueueImpl>::EndExecute(TEnqueuedAction* action) auto timeFromStart = CpuDurationToDuration(action->FinishedAt - action->StartedAt); auto timeFromEnqueue = CpuDurationToDuration(action->FinishedAt - action->EnqueuedAt); - auto updateCounters = [&] (const TCountersPtr& counters) { - if (counters) { - counters->ExecTimer.Record(timeFromStart); - counters->CumulativeTimeCounter.Add(timeFromStart); - counters->TotalTimer.Record(timeFromEnqueue); - counters->ActiveCallbacks -= 1; - } - }; - updateCounters(Counters_[action->ProfilingTag]); - updateCounters(CumulativeCounters_); + if (Counters_[action->ProfilingTag]) { + Counters_[action->ProfilingTag]->ExecTimer.Record(timeFromStart); + Counters_[action->ProfilingTag]->CumulativeTimeCounter.Add(timeFromStart); + Counters_[action->ProfilingTag]->TotalTimer.Record(timeFromEnqueue); + Counters_[action->ProfilingTag]->ActiveCallbacks -= 1; + } } template <class TQueueImpl> @@ -586,9 +569,9 @@ void TInvokerQueue<TQueueImpl>::RegisterWaitTimeObserver(TWaitTimeObserver waitT } template <class TQueueImpl> -typename TInvokerQueue<TQueueImpl>::TCountersPtr TInvokerQueue<TQueueImpl>::CreateCounters(const TTagSet& tagSet) +typename TInvokerQueue<TQueueImpl>::TCountersPtr TInvokerQueue<TQueueImpl>::CreateCounters(const TTagSet& tagSet, NProfiling::IRegistryImplPtr registry) { - auto profiler = TProfiler("/action_queue").WithTags(tagSet).WithHot(); + auto profiler = TProfiler(registry, "/action_queue").WithTags(tagSet).WithHot(); auto counters = std::make_unique<TCounters>(); counters->EnqueuedCounter = profiler.Counter("/enqueued"); diff --git a/yt/yt/core/concurrency/invoker_queue.h b/yt/yt/core/concurrency/invoker_queue.h index 556a2ea888..48b86df069 100644 --- a/yt/yt/core/concurrency/invoker_queue.h +++ b/yt/yt/core/concurrency/invoker_queue.h @@ -127,13 +127,14 @@ class TInvokerQueue public: TInvokerQueue( TIntrusivePtr<NThreading::TEventCount> callbackEventCount, - const NProfiling::TTagSet& counterTagSet); + const NProfiling::TTagSet& counterTagSet, + NProfiling::IRegistryImplPtr registry = nullptr); TInvokerQueue( TIntrusivePtr<NThreading::TEventCount> callbackEventCount, const std::vector<NProfiling::TTagSet>& counterTagSets, const std::vector<NYTProf::TProfilerTagPtr>& profilerTags, - const NProfiling::TTagSet& cumulativeCounterTagSet); + NProfiling::IRegistryImplPtr registry = nullptr); void SetThreadId(NThreading::TThreadId threadId); @@ -204,14 +205,13 @@ private: using TCountersPtr = std::unique_ptr<TCounters>; std::vector<TCountersPtr> Counters_; - TCountersPtr CumulativeCounters_; std::vector<IInvokerPtr> ProfilingTagSettingInvokers_; std::atomic<bool> IsWaitTimeObserverSet_; TWaitTimeObserver WaitTimeObserver_; - TCountersPtr CreateCounters(const NProfiling::TTagSet& tagSet); + TCountersPtr CreateCounters(const NProfiling::TTagSet& tagSet, NProfiling::IRegistryImplPtr registry); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/profiling_helpers.cpp b/yt/yt/core/concurrency/profiling_helpers.cpp index 4eb9c44c43..b6b67994c0 100644 --- a/yt/yt/core/concurrency/profiling_helpers.cpp +++ b/yt/yt/core/concurrency/profiling_helpers.cpp @@ -28,11 +28,13 @@ TTagSet GetBucketTags( TTagSet GetQueueTags( const TString& threadName, + const TString& bucketName, const TString& queueName) { TTagSet tags; tags.AddTag(std::pair<TString, TString>("thread", threadName)); + tags.AddTag(std::pair<TString, TString>("bucket", bucketName), -1); tags.AddTag(std::pair<TString, TString>("queue", queueName), -1); return tags; diff --git a/yt/yt/core/concurrency/profiling_helpers.h b/yt/yt/core/concurrency/profiling_helpers.h index a5338b32cd..53ab2f6f91 100644 --- a/yt/yt/core/concurrency/profiling_helpers.h +++ b/yt/yt/core/concurrency/profiling_helpers.h @@ -14,6 +14,7 @@ NProfiling::TTagSet GetBucketTags( NProfiling::TTagSet GetQueueTags( const TString& threadName, + const TString& bucketName, const TString& queueName); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp b/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp new file mode 100644 index 0000000000..5f98acb3f0 --- /dev/null +++ b/yt/yt/core/concurrency/unittests/fair_share_action_queue_ut.cpp @@ -0,0 +1,166 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/profiling/timing.h> + +#include <yt/yt/core/concurrency/action_queue.h> +#include <yt/yt/core/concurrency/delayed_executor.h> +#include <yt/yt/core/concurrency/fair_share_action_queue.h> +#include <yt/yt/core/concurrency/scheduler.h> + +#include <yt/yt/core/misc/collection_helpers.h> + +#include <yt/yt/library/profiling/solomon/exporter.h> + +#include <util/datetime/base.h> + +#include <algorithm> +#include <array> +#include <ranges> +#include <utility> + +namespace NYT::NConcurrency { +namespace { + +using namespace NProfiling; + +class TTestFairShareActionQueue + : public ::testing::Test +{ +public: + TSolomonExporterConfigPtr CreateExporterConfig() + { + auto config = New<TSolomonExporterConfig>(); + config->GridStep = TDuration::Seconds(1); + config->EnableCoreProfilingCompatibility = true; + config->EnableSelfProfiling = false; + + return config; + } + + template <class F> + void SyncRunCallback(IInvokerPtr invoker, F&& func) + { + WaitFor(BIND(std::forward<F>(func)).AsyncVia(invoker).Run()).ThrowOnError(); + } + + auto GetSensors(TString json) + { + for (auto& c : json) { + if (c == ':') { + c = '='; + } else if (c == ',') { + c = ';'; + } + } + + auto yson = NYson::TYsonString(json); + + auto list = NYTree::ConvertToNode(yson)->AsMap()->FindChild("sensors"); + + EXPECT_TRUE(list); + + return list->AsList()->GetChildren(); + } + + template <class EQueues, class EBuckets> + void VerifyJson(TString json, THashMap<EBuckets, std::vector<EQueues>> bucketToQueues) + { + TEnumIndexedArray<EQueues, int> enqueuedPerQueue; + TEnumIndexedArray<EBuckets, int> enqueuedPerBucket; + int enqueuedPerThread; + int enqueuedTotal; + + for (const auto& entry : GetSensors(json)) { + auto mapEntry = entry->AsMap(); + auto labels = mapEntry->FindChild("labels")->AsMap(); + + auto sensor = labels->FindChildValue<TString>("sensor"); + + if (!sensor || + sensor != "yt.action_queue.enqueued") + { + continue; + } + + auto value = mapEntry->FindChildValue<int>("value"); + EXPECT_TRUE(value); + + if (auto threadName = labels->FindChildValue<TString>("thread")) { + EXPECT_EQ(threadName, "ActionQueue"); + + if (auto bucketName = labels->FindChildValue<TString>("bucket")) { + if (auto queueName = labels->FindChildValue<TString>("queue")) { + enqueuedPerQueue[TEnumTraits<EQueues>::FromString(*queueName)] = *value; + continue; + } + + enqueuedPerBucket[TEnumTraits<EBuckets>::FromString(*bucketName)] = *value; + continue; + } + + enqueuedPerThread = *value; + } else { + enqueuedTotal = *value; + } + } + + EXPECT_EQ(enqueuedPerThread, enqueuedTotal); + + int totalFromBuckets = 0; + for (auto bucketValue : enqueuedPerBucket) { + totalFromBuckets += bucketValue; + } + EXPECT_EQ(enqueuedPerThread, totalFromBuckets); + + for (auto bucket : TEnumTraits<EBuckets>::GetDomainValues()) { + int totalFromBucketQueues = 0; + + for (auto queue : bucketToQueues[bucket]) { + totalFromBucketQueues += enqueuedPerQueue[queue]; + } + EXPECT_EQ(enqueuedPerBucket[bucket], totalFromBucketQueues); + } + } +}; + +DEFINE_ENUM(EBuckets, + ((Bucket1) (0)) + ((Bucket2) (1)) +); + +DEFINE_ENUM(EQueues, + ((Queue1) (0)) + ((Queue2) (1)) + ((Queue3) (2)) +); + +TEST_F(TTestFairShareActionQueue, TestProfiling) +{ + auto registry = New<TSolomonRegistry>(); + + THashMap<EBuckets, std::vector<EQueues>> bucketToQueues{}; + bucketToQueues[EBuckets::Bucket1] = std::vector{EQueues::Queue1, EQueues::Queue2}; + bucketToQueues[EBuckets::Bucket2] = std::vector{EQueues::Queue3}; + auto queue = CreateEnumIndexedFairShareActionQueue<EQueues>("ActionQueue", bucketToQueues, registry); + + auto config = CreateExporterConfig(); + auto exporter = New<TSolomonExporter>(config, registry); + + exporter->Start(); + + for (auto value : TEnumTraits<EQueues>::GetDomainValues()) { + SyncRunCallback(queue->GetInvoker(value), [] {}); + } + + Sleep(TDuration::Seconds(5)); + + auto json = exporter->ReadJson(); + ASSERT_TRUE(json); + + exporter->Stop(); + + VerifyJson(*json, std::move(bucketToQueues)); +} + +} // namespace +} // namespace NYT::NConcurrency diff --git a/yt/yt/core/concurrency/unittests/ya.make b/yt/yt/core/concurrency/unittests/ya.make index b8cad1eafd..4abc7ebc0e 100644 --- a/yt/yt/core/concurrency/unittests/ya.make +++ b/yt/yt/core/concurrency/unittests/ya.make @@ -17,6 +17,7 @@ SRCS( coroutines_ut.cpp count_down_latch_ut.cpp delayed_executor_ut.cpp + fair_share_action_queue_ut.cpp fair_share_invoker_pool_ut.cpp fair_share_thread_pool_ut.cpp fair_throttler_ut.cpp diff --git a/yt/yt/core/misc/common.h b/yt/yt/core/misc/common.h index bbc9a9ae02..a9e6aa1bb1 100644 --- a/yt/yt/core/misc/common.h +++ b/yt/yt/core/misc/common.h @@ -9,6 +9,10 @@ #include <library/cpp/yt/memory/new.h> #include <library/cpp/yt/memory/ref_counted.h> +// NB: Must be included after IntrusivePtr<T> for +// template instantiation correctness. +#include <library/cpp/yt/misc/optional.h> + #include <util/datetime/base.h> #include <util/generic/hash.h> diff --git a/yt/yt/core/misc/protobuf_helpers.h b/yt/yt/core/misc/protobuf_helpers.h index fc5453ce52..893c387076 100644 --- a/yt/yt/core/misc/protobuf_helpers.h +++ b/yt/yt/core/misc/protobuf_helpers.h @@ -2,7 +2,6 @@ #include "guid.h" #include "mpl.h" -#include "optional.h" #include "object_pool.h" #include "range.h" #include "serialize.h" @@ -14,6 +13,7 @@ #include <library/cpp/yt/memory/ref.h> +#include <library/cpp/yt/misc/optional.h> #include <library/cpp/yt/misc/preprocessor.h> #include <google/protobuf/message.h> diff --git a/yt/yt/core/yson/pull_parser-inl.h b/yt/yt/core/yson/pull_parser-inl.h index 48a0ea9203..e5ad9f8d06 100644 --- a/yt/yt/core/yson/pull_parser-inl.h +++ b/yt/yt/core/yson/pull_parser-inl.h @@ -6,9 +6,10 @@ #include "detail.h" -#include <yt/yt/core/misc/optional.h> #include <yt/yt/core/misc/parser_helpers.h> +#include <library/cpp/yt/misc/optional.h> + #include <optional> namespace NYT::NYson { diff --git a/yt/yt/core/ytree/attributes.h b/yt/yt/core/ytree/attributes.h index c703bfa529..0c99076f19 100644 --- a/yt/yt/core/ytree/attributes.h +++ b/yt/yt/core/ytree/attributes.h @@ -4,7 +4,7 @@ #include <yt/yt/core/yson/string.h> -#include <yt/yt/core/misc/optional.h> +#include <library/cpp/yt/misc/optional.h> namespace NYT::NYTree { diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h index 046a66188d..d6750c6970 100644 --- a/yt/yt/core/ytree/yson_struct_detail.h +++ b/yt/yt/core/ytree/yson_struct_detail.h @@ -5,7 +5,8 @@ #include <yt/yt/core/yson/public.h> #include <yt/yt/core/ypath/public.h> #include <yt/yt/core/ytree/public.h> -#include <yt/yt/core/misc/optional.h> + +#include <library/cpp/yt/misc/optional.h> namespace NYT::NYTree { |