diff options
| author | Timofey Koolin <rekby@users.noreply.github.com> | 2024-09-10 16:38:55 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-09-10 16:38:55 +0300 |
| commit | e7a7fd10042b325b2decf7572044f140e059d5b4 (patch) | |
| tree | 8416f66552969891a3f22dd0271aed98fbd62aa4 | |
| parent | a7d2186ccb9393cd6d569baaa3579dbfa4bd2303 (diff) | |
| download | ydb-e7a7fd10042b325b2decf7572044f140e059d5b4.tar.gz | |
golang pg test (#8132)
26 files changed, 1341 insertions, 0 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index a9f79d684e..943cc8aa22 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -101,6 +101,81 @@ ydb/tests/functional/tenants test_dynamic_tenants.py.* ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.* ydb/tests/functional/tenants test_tenants.py.* ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset* +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestArrayValueBackend] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBinaryByteSliceToInt] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBinaryByteSlicetoUUID] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBindError] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCommit] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnListen] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnPing] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnUnlistenAll] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnUnlisten] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnectorWithNoticeHandler_Simple] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnectorWithNotificationHandler_Simple] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestContextCancelBegin] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestContextCancelQuery] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestContextCancelExec] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestQueryCancelRace] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyFromError] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInBinaryError] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInMultipleValues] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInRaiseStmtTrigger] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInStmtAffectedRows] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInTypes] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInWrongType] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyRespLoopConnectionError] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopySyntaxError] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestEmptyQuery] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestEncodeAndParseTs] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestEncodeDecode] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorClass] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorDuringStartup] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorOnExec] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorOnQueryRowSimpleQuery] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorOnQuery] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestExec] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestFormatTsBackend] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestHasCorrectRootGroupPermissions] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestInfinityTimestamp] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestIssue1046] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestIssue1062] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestIssue186] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerFailedQuery] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerListen] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerPing] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerReconnect] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerUnlistenAll] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerUnlisten] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNewConnector_Connect] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNewConnector_Driver] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNewConnector_WorksWithOpenDB] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNotifyExtra] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNullAfterNonNull] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestParseErrorInExtendedQuery] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestPing] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestQueryCancelledReused] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestQueryRowBugWorkaround] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestReconnect] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestReturning] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestRowsResultTag] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestRuntimeParameters] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext/context.Background] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext/context.WithTimeout] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext/context.WithTimeout_exceeded] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext/context.Background] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext/context.WithTimeout] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext/context.WithTimeout_exceeded] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStringWithNul] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimestampWithTimeZone] +ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTxOptions] ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout ydb/core/kqp/ut/query KqpStats.SysViewClientLost ydb/core/kqp/ut/olap KqpOlap.ManyColumnShards diff --git a/.gitignore b/.gitignore index 13059c6198..5f55e83dd6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ !*/ # Unignore all files inside canondata dir !**/canondata/** +# Allow docker files +!Dockerfile /canonization_show_res.log diff --git a/ydb/tests/postgres_integrations/go-libpq/conftest.py b/ydb/tests/postgres_integrations/go-libpq/conftest.py new file mode 100644 index 0000000000..490e879141 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/conftest.py @@ -0,0 +1,6 @@ +import ydb.tests.postgres_integrations.library +import pytest + + +def pytest_collection_finish(session: pytest.Session): + ydb.tests.postgres_integrations.library.pytest_collection_finish(session) diff --git a/ydb/tests/postgres_integrations/go-libpq/data/.gitignore b/ydb/tests/postgres_integrations/go-libpq/data/.gitignore new file mode 100644 index 0000000000..0c2be4fb7d --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/.gitignore @@ -0,0 +1,3 @@ +/exchange/ +/sources/ +/test-result/ diff --git a/ydb/tests/postgres_integrations/go-libpq/data/Dockerfile b/ydb/tests/postgres_integrations/go-libpq/data/Dockerfile new file mode 100644 index 0000000000..a239669db6 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/Dockerfile @@ -0,0 +1,15 @@ +# For docker context at root git directory + +FROM golang:1.20 + +WORKDIR /project/sources/ + +COPY patch.diff /patch.diff +COPY docker-init.bash /docker-init.bash +RUN /docker-init.bash + +COPY common-go-scripts/go-run-separate-tests.bash /go-run-separate-tests.bash + +COPY docker-start.bash /docker-start.bash + +CMD [ "/docker-start.bash" ] diff --git a/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash new file mode 100755 index 0000000000..f4b5cb5338 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eu + +ONE_TEST_TIMEOUT=5s +TEST_BINARY=./test.binary + +echo "Get test list" +TESTS=$($TEST_BINARY --test.list "^Test" | sort) + + +echo "Shell $SHELL" + +rm -f /test-result/raw/result.txt +for TEST_NAME in $TESTS; do + echo -n "Test: $TEST_NAME " + if echo "$TEST_NAME" | grep -Eq "$YDB_PG_TESTFILTER"; then + echo start + else + echo skip + continue + fi + CMD="$TEST_BINARY --test.run '^$TEST_NAME\$' --test.v --test.timeout='$ONE_TEST_TIMEOUT'" + echo "$CMD" + bash -c "$CMD" >> /test-result/raw/result.txt 2>&1 || true +done + +go-junit-report < /test-result/raw/result.txt > /test-result/raw/result.xml diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml new file mode 100644 index 0000000000..cfbb3d4d48 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml @@ -0,0 +1,22 @@ +version: "3" +services: + project: + network_mode: host + + image: ydb-test/go-pqlib + build: + context: ../../.. + dockerfile: languages/go/libpq/Dockerfile + network: host + environment: + - PGUSER=${YDB_PG_USER:-root} + - PGPASSWORD=${YDB_PG_PASSWORD:-1234} + - PGHOST=${YDB_PG_HOST:-ydb} + - PGPORT=${YDB_PG_PORT:-5432} + - PGDATABASE=${YDB_PG_DATABASE:-local} + - PQGOSSLTESTS=0 + - PQSSLCERTTEST_PATH=certs + - YDB_PG_TESTNAME=${YDB_PG_TESTNAME:-} + volumes: + - ./exchange:/exchange + - ./test-result/:/test-result diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml new file mode 100644 index 0000000000..7c5fb48a15 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml @@ -0,0 +1,40 @@ +version: "3" +services: + ydb: + image: ghcr.io/ydb-platform/local-ydb:nightly + environment: + - "YDB_DEFAULT_LOG_LEVEL=DEBUG" + - "GRPC_TLS_PORT=2135" + - "GRPC_PORT=2136" + - "MON_PORT=8765" + - "YDB_USE_IN_MEMORY_PDISKS=true" + - "POSTGRES_USER=${YDB_PG_USER:-root}" + - "POSTGRES_PASSWORD=${YDB_PG_PASSWORD:-1234}" + - "YDB_FEATURE_FLAGS=enable_temp_tables" + - "YDB_TABLE_ENABLE_PREPARED_DDL=true" + healthcheck: + test: "/bin/sh /health_check" + interval: 1s + start_period: 1m + project: + depends_on: + ydb: + condition: service_healthy + + image: ydb-test/go-pqlib + build: + context: ../../.. + dockerfile: languages/go/libpq/Dockerfile + network: host + environment: + - PGUSER=${YDB_PG_USER:-root} + - PGPASSWORD=${YDB_PG_PASSWORD:-1234} + - PGHOST=${YDB_PG_HOST:-ydb} + - PGPORT=${YDB_PG_PORT:-5432} + - PGDATABASE=${YDB_PG_DATABASE:-/local} + - PQGOSSLTESTS=0 + - PQSSLCERTTEST_PATH=certs + - YDB_PG_TESTNAME=${YDB_PG_TESTNAME:-} + volumes: + - ./exchange:/exchange + - ./test-result/:/test-result diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash new file mode 100755 index 0000000000..9a8c5eff9a --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash @@ -0,0 +1,24 @@ +#!/bin/bash + +set -eu + +apt-get update && apt-get install -y patch + +go install github.com/jstemmer/go-junit-report/v2@v2.0.0 + +mkdir -p /original-sources +cd /original-sources + +wget https://github.com/lib/pq/archive/refs/tags/v1.10.9.tar.gz -O libpq.tar.gz +tar --strip-components=1 -zxvf libpq.tar.gz +rm -f libpq.tar.gz + +mkdir -p /project/sources/ +cp -R /original-sources/. /project/sources/ + +cd /project/sources/ +[ -e /patch.diff ] && patch -s -p0 < /patch.diff + +# cache binary +echo "Build test binary" +go test -c -o ./test.binary diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash new file mode 100755 index 0000000000..5b46ae2049 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash @@ -0,0 +1,36 @@ +#!/bin/bash + +set -eu + +echo "Start script" + +rm -rf /test-result 2> /dev/null || true + +mkdir -p /exchange +mkdir -p /test-result/raw + +if [ -e /exchange/sources ]; then + echo "Skip prepare sources, because it is exist" +else + echo "Copy sources" + mkdir -p /exchange/sources + cp -R /project/sources/. /exchange/sources + chmod -R a+rw /exchange/sources +fi + +cd /project/sources/ + +export YDB_PG_TESTFILTER="${YDB_PG_TESTFILTER:-}" # set YDB_PG_TESTNAME to empty string if it not set + +echo "Run tests: '$YDB_PG_TESTFILTER'" + +echo "Start test" + +mkdir -p /test-result/raw +PQTEST_BINARY_PARAMETERS=no /go-run-separate-tests.bash + +if [ -n "${YDB_PG_TESTFILTER:-}" ]; then + cat /test-result/raw/result.txt +fi + +chmod -R a+rw /test-result diff --git a/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt b/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt new file mode 100644 index 0000000000..469bfc1347 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt @@ -0,0 +1,236 @@ +Test64BitErrorChecking +TestAppendEncodedText +TestAppendEscapedText +TestAppendEscapedTextExistingBuffer +TestArrayScanBackend +TestArrayScanner +TestArrayValueBackend +TestArrayValuer +TestBadConn +TestBinaryByteSliceToInt +TestBinaryByteSlicetoUUID +TestBindError +TestBoolArrayScanBytes +TestBoolArrayScanEmpty +TestBoolArrayScanError +TestBoolArrayScanNil +TestBoolArrayScanString +TestBoolArrayScanUnsupported +TestBoolArrayValue +TestByteSliceToText +TestByteaArrayScanBytes +TestByteaArrayScanEmpty +TestByteaArrayScanError +TestByteaArrayScanNil +TestByteaArrayScanString +TestByteaArrayScanUnsupported +TestByteaArrayValue +TestByteaOutputFormatEncoding +TestByteaOutputFormats +TestCloseBadConn +TestCommit +TestCommitInFailedTransaction +TestCommitInFailedTransactionWithCancelContext +TestConnClose +TestConnExecDeadlock +TestConnListen +TestConnPing +TestConnPrepareContext +TestConnPrepareContext/context.Background +TestConnPrepareContext/context.WithTimeout +TestConnPrepareContext/context.WithTimeout_exceeded +TestConnUnlisten +TestConnUnlistenAll +TestConnectorWithNoticeHandler_Simple +TestConnectorWithNotificationHandler_Simple +TestContextCancelBegin +TestContextCancelExec +TestContextCancelQuery +TestCopyFromError +TestCopyInBinaryError +TestCopyInMultipleValues +TestCopyInRaiseStmtTrigger +TestCopyInSchemaStmt +TestCopyInStmt +TestCopyInStmtAffectedRows +TestCopyInTypes +TestCopyInWrongType +TestCopyOutsideOfTxnError +TestCopyRespLoopConnectionError +TestCopySyntaxError +TestDataType +TestDataTypeLength +TestDataTypeName +TestDataTypePrecisionScale +TestDecodeBool +TestDecodeUUIDBackend +TestDecodeUUIDBinaryError +TestEmptyQuery +TestEmptyResultSetColumns +TestEncodeAndParseTs +TestEncodeDecode +TestErrorClass +TestErrorDuringStartup +TestErrorDuringStartupClosesConn +TestErrorOnExec +TestErrorOnQuery +TestErrorOnQueryRowSimpleQuery +TestErrorSQLState +TestExec +TestFloat32ArrayScanBytes +TestFloat32ArrayScanEmpty +TestFloat32ArrayScanError +TestFloat32ArrayScanNil +TestFloat32ArrayScanString +TestFloat32ArrayScanUnsupported +TestFloat32ArrayValue +TestFloat64ArrayScanBytes +TestFloat64ArrayScanEmpty +TestFloat64ArrayScanError +TestFloat64ArrayScanNil +TestFloat64ArrayScanString +TestFloat64ArrayScanUnsupported +TestFloat64ArrayValue +TestFormatAndParseTimestamp +TestFormatTs +TestFormatTsBackend +TestFullParseURL +TestGenericArrayScanDelimiter +TestGenericArrayScanErrors +TestGenericArrayScanScannerArrayBytes +TestGenericArrayScanScannerArrayString +TestGenericArrayScanScannerSliceBytes +TestGenericArrayScanScannerSliceEmpty +TestGenericArrayScanScannerSliceNil +TestGenericArrayScanScannerSliceString +TestGenericArrayScanUnsupported +TestGenericArrayValue +TestGenericArrayValueErrors +TestGenericArrayValueUnsupported +TestHasCorrectRootGroupPermissions +TestIPv6LoopbackParseURL +TestInfinityTimestamp +TestInt32ArrayScanBytes +TestInt32ArrayScanEmpty +TestInt32ArrayScanError +TestInt32ArrayScanNil +TestInt32ArrayScanString +TestInt32ArrayScanUnsupported +TestInt32ArrayValue +TestInt64ArrayScanBytes +TestInt64ArrayScanEmpty +TestInt64ArrayScanError +TestInt64ArrayScanNil +TestInt64ArrayScanString +TestInt64ArrayScanUnsupported +TestInt64ArrayValue +TestInvalidProtocolParseURL +TestIsUTF8 +TestIssue1046 +TestIssue1062 +TestIssue186 +TestIssue196 +TestIssue282 +TestIssue494 +TestIssue617 +TestListenerClose +TestListenerConnCloseWhileQueryIsExecuting +TestListenerFailedQuery +TestListenerListen +TestListenerPing +TestListenerReconnect +TestListenerUnlisten +TestListenerUnlistenAll +TestMinimalURL +TestMultipleEmptyResult +TestMultipleResult +TestMultipleSimpleQuery +TestNewConnector_Connect +TestNewConnector_Driver +TestNewConnector_WorksWithOpenDB +TestNewListenerConn +TestNoData +TestNotifyExtra +TestNullAfterNonNull +TestOpenURL +TestParameterCountMismatch +TestParseArray +TestParseArrayError +TestParseComplete +TestParseEnviron +TestParseErrorInExtendedQuery +TestParseOpts +TestParseTs +TestParseTsErrors +TestPgpass +TestPing +TestQueryCancelRace +TestQueryCancelledReused +TestQueryRowBugWorkaround +TestQuickClose +TestQuoteIdentifier +TestQuoteLiteral +TestReadFloatPrecision +TestReconnect +TestReturning +TestRowsCloseBeforeDone +TestRowsColumnTypes +TestRowsResultTag +TestRuntimeParameters +TestSNISupport +TestSNISupport/SNI_is_not_passed_when_disabled +TestSNISupport/SNI_is_not_set_for_IPv4 +TestSNISupport/SNI_is_passed_when_asked_for +TestSNISupport/SNI_is_set_by_default +TestSSLClientCertificates +TestSSLConnection +TestSSLRequireWithRootCert +TestSSLVerifyCA +TestSSLVerifyFull +TestScanNilTimestamp +TestScanTimestamp +TestSimpleParseURL +TestSimpleQuery +TestStatment +TestStmtExecContext +TestStmtExecContext/context.Background +TestStmtExecContext/context.WithTimeout +TestStmtExecContext/context.WithTimeout_exceeded +TestStmtQueryContext +TestStmtQueryContext/context.Background +TestStmtQueryContext/context.WithTimeout +TestStmtQueryContext/context.WithTimeout_exceeded +TestStringArrayScanBytes +TestStringArrayScanEmpty +TestStringArrayScanError +TestStringArrayScanNil +TestStringArrayScanString +TestStringArrayScanUnsupported +TestStringArrayValue +TestStringToBytea +TestStringToUUID +TestStringWithNul +TestTextByteSliceToInt +TestTextByteSliceToUUID +TestTextDecodeIntoString +TestTimeWithTimezone +TestTimeWithTimezone/11:59:59+00:00_=>_0000-01-01T11:59:59Z +TestTimeWithTimezone/11:59:59+04:00_=>_0000-01-01T11:59:59+04:00 +TestTimeWithTimezone/11:59:59+04:01:02_=>_0000-01-01T11:59:59+04:01 +TestTimeWithTimezone/11:59:59-04:01:02_=>_0000-01-01T11:59:59-04:01 +TestTimeWithTimezone/24:00+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00 +TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone +TestTimeWithoutTimezone/11:59:59_=>_0000-01-01T11:59:59Z +TestTimeWithoutTimezone/24:00:00.000000_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone/24:00:00.0_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone/24:00:00_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone/24:00_=>_0000-01-02T00:00:00Z +TestTimestampWithOutTimezone +TestTimestampWithTimeZone +TestTxOptions +TestXactMultiStmt diff --git a/ydb/tests/postgres_integrations/go-libpq/data/patch.diff b/ydb/tests/postgres_integrations/go-libpq/data/patch.diff new file mode 100644 index 0000000000..32c1d98280 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/patch.diff @@ -0,0 +1,227 @@ +diff -ruN /original-sources/conn_test.go ./conn_test.go +--- /original-sources/conn_test.go 2023-04-26 04:34:24.000000000 +0000 ++++ ./conn_test.go 2023-09-15 09:16:17.844086739 +0000 +@@ -230,7 +230,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err := db.Exec("CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -318,7 +318,7 @@ + + if !r1.Next() { + if r.Err() != nil { +- t.Fatal(r1.Err()) ++ t.Fatal(r.Err()) + } + t.Fatal("expected row") + } +@@ -862,7 +862,7 @@ + defer db.Close() + + // stmt.exec() +- _, err := db.Exec("CREATE TEMP TABLE notnulltemp (a varchar(10) not null)") ++ _, err := db.Exec("CREATE TEMP TABLE notnulltemp (a varchar(10) not null, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -973,7 +973,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("create temp table test (i integer)") ++ _, err := db.Exec("create temp table test (i integer, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -1014,7 +1014,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE distributors (did integer default 0, dname text)") ++ _, err := db.Exec("CREATE TEMP TABLE distributors (did integer default 0, dname text, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -1067,7 +1067,7 @@ + } + defer txn.Rollback() + +- rows, err := txn.Query("CREATE TEMP TABLE foo(f1 int)") ++ rows, err := txn.Query("CREATE TEMP TABLE foo(f1 int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -1290,7 +1290,7 @@ + + if !r.Next() { + if r.Err() != nil { +- t.Fatal(err) ++ t.Fatal(r.Err()) + } + t.Fatal("expected row") + } +@@ -1305,7 +1305,7 @@ + + if !r.Next() { + if r.Err() != nil { +- t.Fatal(err) ++ t.Fatal(r.Err()) + } + t.Fatal("expected row") + } +@@ -1351,11 +1351,11 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err := db.Exec("CREATE TEMP TABLE temp (a int primary key)") + if err != nil { + t.Fatal(err) + } +- sqlInsert := "INSERT INTO temp VALUES (1)" ++ sqlInsert := "INSERT INTO temp (a) VALUES (1)" + sqlSelect := "SELECT * FROM temp" + tx, err := db.Begin() + if err != nil { +@@ -1501,7 +1501,7 @@ + } + + value, success := tryGetParameterValue() +- if success != test.success && !test.success { ++ if success != test.success && !success { + t.Fatalf("%v: unexpected error: %v", test.conninfo, err) + } + if success != test.success { +@@ -1603,7 +1603,7 @@ + ra int64 + }{ + { +- query: "CREATE TEMP TABLE temp (a int)", ++ query: "CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)", + tag: "CREATE TABLE", + }, + { +@@ -1623,19 +1623,19 @@ + }, + // Multiple statements that don't return rows should return the last tag. + { +- query: "CREATE TEMP TABLE t (a int); DROP TABLE t", ++ query: "CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); DROP TABLE t", + tag: "DROP TABLE", + }, + // Ensure a rows-returning query in any position among various tags-returing + // statements will prefer the rows. + { +- query: "SELECT 1; CREATE TEMP TABLE t (a int); DROP TABLE t", ++ query: "SELECT 1; CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); DROP TABLE t", + }, + { +- query: "CREATE TEMP TABLE t (a int); SELECT 1; DROP TABLE t", ++ query: "CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); SELECT 1; DROP TABLE t", + }, + { +- query: "CREATE TEMP TABLE t (a int); DROP TABLE t; SELECT 1", ++ query: "CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); DROP TABLE t; SELECT 1", + }, + } + +@@ -1775,7 +1775,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err := db.Exec("CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +diff -ruN /original-sources/copy_test.go ./copy_test.go +--- /original-sources/copy_test.go 2023-04-26 04:34:24.000000000 +0000 ++++ ./copy_test.go 2023-09-15 09:14:56.207034622 +0000 +@@ -56,7 +56,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -125,7 +125,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -195,7 +195,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, text VARCHAR, blob BYTEA, nothing VARCHAR)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, text VARCHAR, blob BYTEA, nothing VARCHAR, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -254,7 +254,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -302,7 +302,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -327,7 +327,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -383,7 +383,7 @@ + t.Fatal(err) + } + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -463,7 +463,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar, _stub_id Serial PRIMARY KEY)") + if err != nil { + b.Fatal(err) + } +diff -ruN /original-sources/issues_test.go ./issues_test.go +--- /original-sources/issues_test.go 2023-04-26 04:34:24.000000000 +0000 ++++ ./issues_test.go 2023-08-22 09:35:23.189760257 +0000 +@@ -113,7 +113,7 @@ + time.Sleep(10 * time.Millisecond) + cancel() + }() +- row := db.QueryRowContext(ctx, "select pg_sleep(0.5)") ++ row := db.QueryRowContext(ctx, "select pg_sleep(4::float8)") + var pgSleepVoid string + err := row.Scan(&pgSleepVoid) + if pgErr := (*Error)(nil); !(errors.As(err, &pgErr) && pgErr.Code == cancelErrorCode) { diff --git a/ydb/tests/postgres_integrations/go-libpq/data/run-test.bash b/ydb/tests/postgres_integrations/go-libpq/data/run-test.bash new file mode 100755 index 0000000000..0a9e4945a0 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/run-test.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# https://github.com/lib/pq + + +set -eu + +LOCAL_DIR=$(dirname "$0") +LOCAL_DIR=$(realpath "$LOCAL_DIR") + +scripts/run-test.bash "$LOCAL_DIR" diff --git a/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt b/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt new file mode 100644 index 0000000000..1ca9981b7b --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt @@ -0,0 +1,2 @@ +# ydb segfalts +TestIssue494 # https://github.com/ydb-platform/ydb/issues/8410 diff --git a/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt b/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt new file mode 100644 index 0000000000..e0b282854c --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt @@ -0,0 +1,106 @@ +TestAppendEncodedText +TestAppendEscapedText +TestAppendEscapedTextExistingBuffer +TestArrayScanner +TestArrayValuer +TestBadConn +TestBoolArrayScanBytes +TestBoolArrayScanEmpty +TestBoolArrayScanError +TestBoolArrayScanNil +TestBoolArrayScanString +TestBoolArrayScanUnsupported +TestBoolArrayValue +TestByteaArrayScanBytes +TestByteaArrayScanEmpty +TestByteaArrayScanError +TestByteaArrayScanNil +TestByteaArrayScanString +TestByteaArrayScanUnsupported +TestByteaArrayValue +TestByteaOutputFormatEncoding +TestCloseBadConn +TestConnPrepareContext/context.WithTimeout_exceeded +TestCopyInSchemaStmt +TestCopyInStmt +TestDataType +TestDataTypeLength +TestDataTypeName +TestDataTypePrecisionScale +TestDecodeUUIDBinaryError +TestErrorDuringStartup +TestErrorDuringStartupClosesConn +TestErrorSQLState +TestFloat32ArrayScanBytes +TestFloat32ArrayScanEmpty +TestFloat32ArrayScanError +TestFloat32ArrayScanNil +TestFloat32ArrayScanString +TestFloat32ArrayScanUnsupported +TestFloat32ArrayValue +TestFloat64ArrayScanBytes +TestFloat64ArrayScanEmpty +TestFloat64ArrayScanError +TestFloat64ArrayScanNil +TestFloat64ArrayScanString +TestFloat64ArrayScanUnsupported +TestFloat64ArrayValue +TestFormatAndParseTimestamp +TestFormatTs +TestFullParseURL +TestGenericArrayScanDelimiter +TestGenericArrayScanErrors +TestGenericArrayScanScannerArrayBytes +TestGenericArrayScanScannerArrayString +TestGenericArrayScanScannerSliceBytes +TestGenericArrayScanScannerSliceEmpty +TestGenericArrayScanScannerSliceNil +TestGenericArrayScanScannerSliceString +TestGenericArrayScanUnsupported +TestGenericArrayValue +TestGenericArrayValueErrors +TestGenericArrayValueUnsupported +TestIPv6LoopbackParseURL +TestInt32ArrayScanBytes +TestInt32ArrayScanEmpty +TestInt32ArrayScanError +TestInt32ArrayScanNil +TestInt32ArrayScanString +TestInt32ArrayScanUnsupported +TestInt32ArrayValue +TestInt64ArrayScanBytes +TestInt64ArrayScanEmpty +TestInt64ArrayScanError +TestInt64ArrayScanNil +TestInt64ArrayScanString +TestInt64ArrayScanUnsupported +TestInt64ArrayValue +TestInvalidProtocolParseURL +TestIsUTF8 +TestMinimalURL +TestParseArray +TestParseArrayError +TestParseComplete +TestParseEnviron +TestParseOpts +TestParseTs +TestParseTsErrors +TestQuoteIdentifier +TestQuoteLiteral +TestSNISupport +TestSNISupport/SNI_is_not_passed_when_disabled +TestSNISupport/SNI_is_not_set_for_IPv4 +TestSNISupport/SNI_is_passed_when_asked_for +TestSNISupport/SNI_is_set_by_default +TestScanNilTimestamp +TestScanTimestamp +TestSimpleParseURL +TestStringArrayScanBytes +TestStringArrayScanEmpty +TestStringArrayScanError +TestStringArrayScanNil +TestStringArrayScanString +TestStringArrayScanUnsupported +TestStringArrayValue +TestStringWithNul +TestTextDecodeIntoString diff --git a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py new file mode 100644 index 0000000000..08736e56ee --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py @@ -0,0 +1,32 @@ +# from .conftest import integrations +import typing + +import pytest +import yatest + +from ydb.tests.postgres_integrations import library as tl + + +def filter_formatter(test_names: typing.List[str]) -> str: + return "^(" + "|".join(test_names) + ")$" + + +def setup_module(module: pytest.Module): + tl.setup_module(module) + + +def teardown_module(module: pytest.Module): + tl.teardown_module(module) + + +def test_pg_generated(testname): + tl.execute_test(testname) + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + if metafunc.definition.name == "test_pg_generated": + tl.pytest_generate_tests(metafunc) + + +tl.set_filter_formatter(filter_formatter) +tl.set_tests_folder(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) diff --git a/ydb/tests/postgres_integrations/go-libpq/ya.make b/ydb/tests/postgres_integrations/go-libpq/ya.make new file mode 100644 index 0000000000..279d12a65f --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/ya.make @@ -0,0 +1,56 @@ +PY3TEST() + +FORK_TEST_FILES() +TIMEOUT(600) + + +# copy from https://docs.yandex-team.ru/devtools/test/environment#docker-compose +REQUIREMENTS( + container:4467981730 # container with docker + cpu:all dns:dns64 +) + +IF(OPENSOURCE) + SIZE(MEDIUM) # for run per PR + + # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, + # which makes it impossible to run them during precommit checks on Github CI. + # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928. + + SET(TEST_TAGS_VALUE) + SET(TEST_REQUIREMENTS_VALUE) + # This requirement forces tests to be launched consequently, + # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. + # See DEVTOOLSSUPPORT-44103, YA-1759 for details. + TAG(ya:not_autocheck) +ELSE() + SIZE(LARGE) # run in sandbox with timeout more than a minute + TAG( + ya:external + ya:fat + ya:force_sandbox + ) +ENDIF() + + +ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") +ENV(YDB_ALLOCATE_PGWIRE_PORT="true") +DEPENDS( + ydb/apps/ydbd +) + +TEST_SRCS( + conftest.py + docker_wrapper_test.py +) + + +DATA( + arcadia/ydb/tests/postgres_integrations/go-libpq/data +) + +PEERDIR( + ydb/tests/postgres_integrations/library +) + +END() diff --git a/ydb/tests/postgres_integrations/library/__init__.py b/ydb/tests/postgres_integrations/library/__init__.py new file mode 100644 index 0000000000..3d978f45bd --- /dev/null +++ b/ydb/tests/postgres_integrations/library/__init__.py @@ -0,0 +1,13 @@ +__all__ = [ # noqa + 'IntegrationTests', + 'PgTestWrapper', + 'pytest_collection_finish', + 'set_filter_formatter', + 'set_tests_folder', + 'setup_module', + 'teardown_module', + 'execute_test', + 'pytest_generate_tests', +] + +from .pytest_integration import * # noqa diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py new file mode 100644 index 0000000000..dda063b96c --- /dev/null +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -0,0 +1,305 @@ +import os +import shutil + +from typing import Callable, Dict, List, Set, Optional, Union +from os import path +from dataclasses import dataclass +from enum import Enum + +import docker +import xmltodict +import pytest + +import yatest + +import logging + +from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory +from ydb.tests.library.harness.kikimr_runner import KiKiMR + + +class TestState(Enum): + PASSED = 1 + FAILED = 2 + SKIPPED = 3 + + +@dataclass +class TestCase: + name: str + state: TestState + log: str + + +_tests_for_run_in_docker: pytest.Session = [] +_filter_format_function = Callable[[List[str]], str] +_filter_formatter: Optional[_filter_format_function] = None +_tests_folder: Optional[str] = None +_test_results: Optional[Dict[str, TestCase]] = None +_kikimr_factory: KiKiMR = kikimr_cluster_factory() +_integration_tests: Optional[List[str]] = None +_skip_tests: Dict[str, str] = dict() # [test name: reason] + + +def pytest_collection_finish(session: pytest.Session): + global _tests_for_run_in_docker + + print("rekby set selected items: ", session.items) + selected_tests = [] + for item in session.items: + print(f"rekby, selected item name: '{item.name}'", ) + if item.name.startswith("test_pg_generated["): + print("rekby selected test item:", item) + test_name = item.callspec.id + print(f"rekby selected test: {test_name}") + selected_tests.append(test_name) + selected_tests.sort() + print("rekby: result selected tests", selected_tests) + _tests_for_run_in_docker = list() + for test in selected_tests: + if test not in _skip_tests: + _tests_for_run_in_docker.append(test) + print("rekby, tests for run", _tests_for_run_in_docker) + + +def set_filter_formatter(f: _filter_format_function): + global _filter_formatter + _filter_formatter = f + + +def set_tests_folder(folder: str): + global _tests_folder, _integration_tests, _skip_tests + print("rekby, set_tests_folder called") + _tests_folder = folder + _integration_tests = _read_integration_tests(folder) + _skip_tests = _read_skip_tests(folder) + + +def setup_module(module: pytest.Module): + if len(_tests_for_run_in_docker) == 0: + return + + global _test_results + try: + exchange_folder = path.join(yatest.common.output_path(), "exchange") + os.mkdir(exchange_folder) + except FileExistsError: + pass + + tests_result_folder = path.join(yatest.common.output_path(), "test-result") + shutil.rmtree(tests_result_folder, ignore_errors=True) + os.mkdir(tests_result_folder) + + image = _docker_build(_tests_folder) + + pg_port = _run_ydb() + env = _prepare_docker_env(pg_port, _tests_for_run_in_docker) + _run_tests_in_docker(image, env, exchange_folder, tests_result_folder) + + test_results_file = path.join(tests_result_folder, "raw", "result.xml") + _test_results = _read_tests_result(test_results_file) + + +def teardown_module(module): + """teardown any state that was previously setup with a setup_module + method. + """ + _stop_ydb() + + +def _run_ydb() -> int: + """ + Run YDB cluster and return pgwire port number. + """ + _kikimr_factory.start() + node = _kikimr_factory.nodes[1] + print("rekby: pgwire port", node.pgwire_port) + return node.pgwire_port + + +def _stop_ydb(): + _kikimr_factory.stop() + + +def _prepare_docker_env(pgwire_port: str, test_names: List[str]) -> List[str]: + test_filter = _filter_formatter(test_names) + return [ + "PGUSER=root", + "PGPASSWORD=1234", + "PGHOST=localhost", + f"PGPORT={pgwire_port}", + "PGDATABASE=/Root", + "PQGOSSLTESTS=0", + "PQSSLCERTTEST_PATH=certs", + f"YDB_PG_TESTFILTER={test_filter}", + ] + + +def _docker_build(folder: str) -> str: + image_name = 'ydb-pg-test-image' + logging.debug(f"rekby, docker folder: '{folder}'") + + import glob + files_list = glob.glob(folder + "/data/*") + logging.debug(f"rekby, {folder}/data/ contents: {files_list}") + + client: docker.Client = docker.from_env() + client.images.build( + path=folder, + tag=image_name, + rm=True, + network_mode='host', + ) + return image_name + + +def _run_tests_in_docker( + image: str, + env: Union[List[str], Dict[str, str]], + exchange_folder: str, + results_folder: str, + ): + + # TODO: run YDB with scripts/receipt and get connection port/database with runtime + client: docker.Client = docker.from_env() + + container = client.containers.create( + image=image, + # command="/docker-start.bash", + # detach=True, + # auto_remove=True, + environment=env, + mounts=[ + docker.types.Mount( + target="/exchange", + source=exchange_folder, + type="bind", + ), + docker.types.Mount( + target="/test-result", + source=results_folder, + type="bind", + ), + ], + network_mode='host', + ) + try: + container.start() + container.wait() + print(container.logs().decode()) + finally: + container.remove() + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + """ + Return tests for run through pytest. + """ + print("rekby, integration tests:", _integration_tests) + metafunc.parametrize('testname', _integration_tests, ids=_integration_tests) + + +def execute_test(testname: str): + if testname in _skip_tests: + pytest.skip(_skip_tests[testname]) + + try: + test = _test_results[testname] + except KeyError: + pytest.fail("test result not found, may be the test was not runned") + + if test.state == TestState.PASSED: + logging.getLogger().log(logging.INFO, test.log) + return + if test.state == TestState.SKIPPED: + logging.getLogger().log(logging.INFO, test.log) + pytest.skip() + if test.state == TestState.FAILED: + logging.getLogger().log(logging.ERROR, test.log) + pytest.fail() + + raise Exception(f"Unexpected test state: '{test.state}'") + + +def _read_integration_tests(folder: str) -> Set[str]: + with open(path.join(folder, "full-test-list.txt"), "rt") as f: + all = set(line.strip() for line in f.readlines()) + + with open(path.join(folder, "unit-tests.txt"), "rt") as f: + unit = set(f.readlines()) + + test_list_for_run = list(all - unit) + test_list_for_run.sort() + return test_list_for_run + + +def _read_skip_tests(folder: str) -> Dict[str, str]: + res = dict() + try: + fpath = path.join(folder, "skip-tests.txt") + with open(fpath) as f: + for line in f.readlines(): + if "# " in line: + line = line[:line.rindex("# ")] + + line = line.strip() + if line == "": + continue + + res[line] = f"skipped by '{fpath}'" + except FileNotFoundError: + pass + + return res + + +def _read_tests_result(filepath: str) -> Dict[str, TestCase]: + with open(filepath, "rt") as f: + data = f.read() + d = xmltodict.parse(data, force_list=("testcase",)) + testsuites = d["testsuites"] + test_suite = testsuites["testsuite"] + test_cases = test_suite["testcase"] + + res: Dict[str, TestCase] = dict() + + def get_text(test_case, field_name: str) -> str: + field_val = test_case[field_name] + if type(field_val) is str: + return field_val + elif type(field_val) is dict: + prefix = field_val.get("@message", "") + "\n" + if prefix == "\n": + prefix = "" + return prefix + field_val.get("#text", "") + raise Exception(f"Unknown field val for field '{field_name}':\n{field_val}") + + for test_case in test_cases: + class_name = test_case["@classname"] + test_name = test_case["@name"] + if class_name == "": + name = test_name + else: + name = test_case["@classname"] + "/" + test_case["@name"] + + print("rekby-debug", test_case) + if "failure" in test_case: + test_state = TestState.FAILED + log = get_text(test_case, "failure") + elif "error" in test_case: + test_state = TestState.FAILED + log = get_text(test_case, "error") + elif "skipped" in test_case: + test_state = TestState.SKIPPED + log = get_text(test_case, "skipped") + else: + test_state = TestState.PASSED + log = "" + + res[name] = TestCase( + name=name, + state=test_state, + log=log, + ) + + return res diff --git a/ydb/tests/postgres_integrations/library/ut/data/junit-results-example.xml b/ydb/tests/postgres_integrations/library/ut/data/junit-results-example.xml new file mode 100644 index 0000000000..6627aaf299 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/data/junit-results-example.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="UTF-8"?> +<testsuites tests="221" failures="112" skipped="6"> + <testsuite name="" tests="221" failures="112" errors="0" id="0" hostname="localhost" skipped="6" time="0.330" timestamp="2024-07-16T15:06:53Z"> + <testcase name="OK" classname="o" time="0.000"></testcase> + <testcase name="failed1" classname="f" time="0.000"> + <failure>fail mess</failure> + </testcase> + <testcase name="failed2" classname="f" time="0.000"> + <failure message="Failed"><![CDATA[escaped error]]></failure> + </testcase> + <testcase name="error1" classname="f" time="0.000"> + <error message="No test result found">panic and timeout</error> + </testcase> + <testcase name="skipped1" classname="s" time="0.000"> + <skipped message="Skipped">skip message</skipped> + </testcase> + <testcase name="skipped2" classname="s" time="0.000"> + <skipped><![CDATA[escaped skip message]]></skipped> + </testcase> + </testsuite> +</testsuites> diff --git a/ydb/tests/postgres_integrations/library/ut/data/junit-results-example1.xml b/ydb/tests/postgres_integrations/library/ut/data/junit-results-example1.xml new file mode 100644 index 0000000000..e0c691d728 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/data/junit-results-example1.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<testsuites tests="221" failures="112" skipped="6"> + <testsuite name="" tests="221" failures="112" errors="0" id="0" hostname="localhost" skipped="6" time="0.330" timestamp="2024-07-16T15:06:53Z"> + <testcase name="test-failed" classname="f" time="0.000"> + <failure message="Failed">failed-mess</failure> + </testcase> + </testsuite> +</testsuites> diff --git a/ydb/tests/postgres_integrations/library/ut/integrations_test.py b/ydb/tests/postgres_integrations/library/ut/integrations_test.py new file mode 100644 index 0000000000..a6caf5b255 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/integrations_test.py @@ -0,0 +1,37 @@ +from os import path + +import pytest +import yatest + +from ydb.tests.postgres_integrations.library import pytest_integration +from ydb.tests.postgres_integrations.library.pytest_integration import TestCase, TestState + + +TEST_DATA_FOLDER = yatest.common.source_path("ydb/tests/postgres_integrations/library/ut/data") + + +@pytest.mark.parametrize( + "test", + [ + TestCase(name="o/OK", state=TestState.PASSED, log=""), + TestCase(name="f/failed1", state=TestState.FAILED, log="fail mess"), + TestCase(name="f/failed2", state=TestState.FAILED, log="Failed\nescaped error"), + TestCase(name="f/error1", state=TestState.FAILED, log="No test result found\npanic and timeout"), + TestCase(name="s/skipped1", state=TestState.SKIPPED, log="Skipped\nskip message"), + TestCase(name="s/skipped2", state=TestState.SKIPPED, log="escaped skip message"), + ], + ids=lambda item: item.name +) +def test_read_jtest_results(test): + filepath = path.join(TEST_DATA_FOLDER, "junit-results-example.xml") + parsed_result = pytest_integration._read_tests_result(filepath) + + parsed_test = parsed_result[test.name] + assert test == parsed_test + + +def test_read_jtest_with_one_result(): + filepath = path.join(TEST_DATA_FOLDER, "junit-results-example1.xml") + parsed_result = pytest_integration._read_tests_result(filepath) + parsed_test = parsed_result["f/test-failed"] + assert parsed_test == TestCase(name="f/test-failed", state=TestState.FAILED, log="Failed\nfailed-mess") diff --git a/ydb/tests/postgres_integrations/library/ut/ya.make b/ydb/tests/postgres_integrations/library/ut/ya.make new file mode 100644 index 0000000000..aa508da816 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/ya.make @@ -0,0 +1,16 @@ +PY3TEST() + +TEST_SRCS( + integrations_test.py +) + + +DATA( + arcadia/ydb/tests/postgres_integrations/library/ut/data +) + +PEERDIR( + ydb/tests/postgres_integrations/library +) + +END() diff --git a/ydb/tests/postgres_integrations/library/ya.make b/ydb/tests/postgres_integrations/library/ya.make new file mode 100644 index 0000000000..86959db621 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ya.make @@ -0,0 +1,16 @@ +PY3_LIBRARY() + + +ALL_PY_SRCS() + +PEERDIR( + contrib/python/docker + contrib/python/xmltodict + ydb/tests/library +) + +END() + +RECURSE_FOR_TESTS( + ut +)
\ No newline at end of file diff --git a/ydb/tests/postgres_integrations/ya.make b/ydb/tests/postgres_integrations/ya.make new file mode 100644 index 0000000000..3839f27803 --- /dev/null +++ b/ydb/tests/postgres_integrations/ya.make @@ -0,0 +1,4 @@ +RECURSE( + go-libpq + library +) diff --git a/ydb/tests/ya.make b/ydb/tests/ya.make index 6ad1bcab59..1b847b097d 100644 --- a/ydb/tests/ya.make +++ b/ydb/tests/ya.make @@ -6,6 +6,7 @@ RECURSE( olap oss perf + postgres_integrations stability supp tools |