diff options
| author | Daniil Cherednik <dan.cherednik@gmail.com> | 2023-06-15 18:24:59 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dan.cherednik@gmail.com> | 2023-06-15 18:26:33 +0300 |
| commit | 068d4453cf9fc68c875eee73f5c637bb076f6a71 (patch) | |
| tree | da3e83fdb9488ea08faa39d8b41916744f9acad7 | |
| parent | 7e7de263d4acbc6eacf92b618bcb5f9049bccc9b (diff) | |
| download | ydb-068d4453cf9fc68c875eee73f5c637bb076f6a71.tar.gz | |
Create stable-23-2 branch
x-stable-origin-commit: 3fd4d58117c143ed9e6b21813ccd9e507d2cd4d3
454 files changed, 16278 insertions, 4525 deletions
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml deleted file mode 100644 index 6b754d2b7a..0000000000 --- a/.github/actions/build/action.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Build -description: Build YDB -inputs: - sanitizer: - required: false - type: string - -runs: - using: "composite" - steps: - - name: Configure for sanitizer - shell: bash - if: inputs.sanitizer - run: | - mkdir -p ../build - patch -p1 < ydb/deploy/patches/0001-sanitizer-build.patch - cd ../build - rm -rf * - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \ - -DCMAKE_CXX_FLAGS="-fsanitize=${{ inputs.sanitizer }} -g -fno-omit-frame-pointer" \ - -DCMAKE_C_FLAGS="-fsanitize=${{ inputs.sanitizer }} -g -fno-omit-frame-pointer" \ - -DCMAKE_EXE_LINKER_FLAGS="-rdynamic" \ - ../ydb - - name: Configure - shell: bash - if: ${{!inputs.sanitizer}} - run: | - mkdir -p ../build - cd ../build - rm -rf * - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \ - -DCMAKE_CXX_FLAGS="-g" \ - -DCMAKE_C_FLAGS="-g" \ - -DCMAKE_EXE_LINKER_FLAGS="-rdynamic" \ - ../ydb - - name: Build - shell: bash - run: | - ccache -z - cd ../build - ninja - ccache -s diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml deleted file mode 100644 index 0a3c848c84..0000000000 --- a/.github/actions/test/action.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: build-and-test -description: Build YDB and run Tests -inputs: - log_suffix: - required: true - type: string - test_label_regexp: - required: false - type: string - aws_key_id: - required: true - type: string - aws_key_value: - required: true - type: string - testman_token: - required: false - type: string - testman_url: - required: false - type: string - testman_project_id: - required: false - type: string - aws_bucket: - required: true - type: string - aws_endpoint: - required: true - type: string - -runs: - using: "composite" - steps: - - name: Init - id: init - shell: bash - run: | - mkdir -p artifacts tmp test_reports - rm -rf artifacts/* tmp/* test_reports/* - echo "WORKDIR=$(pwd)" >> $GITHUB_ENV - echo "TESTREPDIR=$(pwd)/test_reports" >> $GITHUB_ENV - echo "TESTMO_TOKEN=${{inputs.testman_token}}" >> $GITHUB_ENV - echo "TESTMO_URL=${{inputs.testman_url}}" >> $GITHUB_ENV - echo "logfilename=${{inputs.log_suffix}}-ctest-stdout.gz" >> $GITHUB_OUTPUT - - name: Install Node required for Testmo CLI - uses: actions/setup-node@v3 - with: - node-version: 19 - - name: Install Testmo CLI - shell: bash - run: npm install -g @testmo/testmo-cli - - name: Test history run create - id: th - if: inputs.testman_token - shell: bash - env: - PR_NUMBER: ${{ github.event.number }} - run: | - RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" - BRANCH_TAG="$GITHUB_REF_NAME" - - case $GITHUB_EVENT_NAME in - workflow_dispatch) - TESTMO_RUN_NAME="${{ github.run_id }} manual" - EXTRA_TAG="manual" - ;; - pull_request | pull_request_target) - TESTMO_RUN_NAME="${{ github.run_id }} PR #${PR_NUMBER}" - EXTRA_TAG="pr" - BRANCH_TAG="" - ;; - schedule) - TESTMO_RUN_NAME="${{ github.run_id }} schedule" - EXTRA_TAG="schedule" - ;; - *) - TESTMO_RUN_NAME="${{ github.run_id }}" - EXTRA_TAG="" - ;; - esac - - testmo automation:resources:add-link --name build --url $RUN_URL --resources testmo.json - testmo automation:resources:add-field --name git-sha --type string --value ${GITHUB_SHA:0:7} --resources testmo.json - testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{inputs.testman_project_id}} --name "$TESTMO_RUN_NAME" \ - --source "${{inputs.log_suffix}}" --resources testmo.json \ - --tags "$BRANCH_TAG" --tags "$EXTRA_TAG" | \ - echo "runid=$(cat)" >> $GITHUB_OUTPUT - - - name: Test - shell: bash - run: | - cd $WORKDIR/../build/ydb - - echo "Stdout log (gzip archive): ${{inputs.aws_endpoint}}/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{steps.init.outputs.logfilename}}" >> $GITHUB_STEP_SUMMARY - - # Sed removes coloring from the output - - TMPDIR=$WORKDIR/tmp GTEST_OUTPUT="xml:$TESTREPDIR/unittests/" Y_UNITTEST_OUTPUT="xml:$TESTREPDIR/unittests/" \ - ctest -j28 --timeout 1200 --force-new-ctest-process --output-on-failure \ - --output-junit $TESTREPDIR/suites/ctest_report.xml \ - -L '${{inputs.test_label_regexp}}' | \ - sed -e 's/\x1b\[[0-9;]*m//g' | \ - tee >(gzip --stdout > $WORKDIR/artifacts/${{steps.init.outputs.logfilename}}) | \ - grep -E '(Test\s*#.*\*\*\*|\[FAIL\])|.*tests passed,.*tests failed out of' | \ - tee $WORKDIR/short.log - - name: Test history upload results - if: always() && inputs.testman_token - shell: bash - run: | - testmo automation:run:submit-thread \ - --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \ - --results $TESTREPDIR/unittests/*.xml - testmo automation:run:submit-thread \ - --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \ - --results $TESTREPDIR/suites/*.xml \ - -- cat $WORKDIR/short.log - testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} - - name: Upload S3 - uses: shallwefootball/s3-upload-action@master - if: always() - with: - aws_key_id: ${{inputs.AWS_KEY_ID }} - aws_secret_access_key: ${{inputs.AWS_KEY_VALUE}} - aws_bucket: ${{inputs.aws_bucket}} - source_dir: artifacts - destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}' - endpoint: ${{inputs.aws_endpoint}} diff --git a/.github/actions/test_python/action.yml b/.github/actions/test_python/action.yml deleted file mode 100644 index f195e408f2..0000000000 --- a/.github/actions/test_python/action.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: test-python -description: Run functional Python tests -inputs: - log_suffix: - required: true - type: string - test_label_regexp: - required: false - type: string - aws_key_id: - required: true - type: string - aws_key_value: - required: true - type: string - aws_bucket: - required: true - type: string - aws_endpoint: - required: true - type: string - -runs: - using: "composite" - steps: - - name: Test - shell: bash - run: | - export source_root=$(pwd) - export build_root=$(pwd)/../build/ - mkdir -p ../artifacts - rm -rf ../artifacts/* - - echo "Stdout log (gzip archive): https://storage.yandexcloud.net/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-pytest-stdout.gz" >> $GITHUB_STEP_SUMMARY - cd ${source_root}/ydb/tests/functional/ - bad_suites=$(grep -Eo 'ignore=[a-zA-Z_-]*' pytest.ini | sed -e 's/ignore=//g') - suites="" - for suite in $(echo */ | sed -e 's/\///g'); do - if [[ $(echo "$bad_suites" | grep -F -e $suite -) == '' ]]; then - suites+=$suite - suites+=$'\n' - fi - done - if [[ "${{inputs.test_label_regexp}}" != '' ]]; then - suites="${{inputs.test_label_regexp}}" - fi - source ${source_root}/ydb/tests/oss/launch/prepare.sh - echo -n "$suites" | parallel -j32 "pytest -o junit_logging=log -o junit_log_passing_tests=False \ - -v --junit-xml=${source_root}/ydb/tests/functional/test-results/xml/{}/res.xml {}" | \ - sed -e 's/\x1b\[[0-9;]*m//g' | \ - tee >(gzip --stdout > ${source_root}/../artifacts/${{inputs.log_suffix}}-pytest-stdout.gz) | \ - tee -a $GITHUB_STEP_SUMMARY - - name: Upload S3 - uses: shallwefootball/s3-upload-action@master - if: always() - with: - aws_key_id: ${{inputs.aws_key_id}} - aws_secret_access_key: ${{inputs.aws_key_value}} - aws_bucket: ${{inputs.aws_bucket}} - source_dir: ../artifacts - destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}' - endpoint: ${{inputs.aws_endpoint}} diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml deleted file mode 100644 index bc3a01ff34..0000000000 --- a/.github/workflows/build_and_test_ondemand.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Build-and-Test-On-demand - -on: - workflow_call: - inputs: - sanitizer: - required: false - type: string - test_label_regexp: - required: false - type: string - image: - type: string - required: false - default: fd8earpjmhevh8h6ug5o - run_unit_tests: - type: boolean - default: true - run_functional_tests: - type: boolean - default: false - workflow_dispatch: - inputs: - sanitizer: - required: false - type: string - test_label_regexp: - required: false - type: string - image: - type: string - required: false - default: fd8earpjmhevh8h6ug5o - run_unit_tests: - type: boolean - default: true - run_functional_tests: - type: boolean - default: false - -jobs: - - provide-runner: - name: Start self-hosted YC runner - timeout-minutes: 5 - runs-on: ubuntu-latest - outputs: - label: ${{steps.start-yc-runner.outputs.label}} - instance-id: ${{steps.start-yc-runner.outputs.instance-id}} - steps: - - name: Start YC runner - id: start-yc-runner - uses: yc-actions/yc-github-runner@v1 - with: - mode: start - yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }} - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - folder-id: ${{secrets.YC_FOLDER}} - image-id: ${{inputs.image}} - disk-size: 930GB - disk-type: network-ssd-nonreplicated - cores: 32 - memory: 64GB - core-fraction: 100 - zone-id: ru-central1-b - subnet-id: ${{secrets.YC_SUBNET}} - - prepare-vm: - name: Prepare runner - uses: ./.github/workflows/prepare_vm_for_build.yml - needs: provide-runner - with: - runner_label: ${{ needs.provide-runner.outputs.label }} - secrets: - rc_auth: ${{ secrets.REMOTE_CACHE_AUTH }} - - main: - name: Build and test - needs: - - provide-runner - - prepare-vm - runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ] - steps: - - name: Checkout PR - uses: actions/checkout@v3 - if: github.event.pull_request.head.sha != '' - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Checkout - uses: actions/checkout@v3 - if: github.event.pull_request.head.sha == '' - - name: Build - id: build - uses: ./.github/actions/build - with: - sanitizer: ${{ inputs.sanitizer }} - - name: Run unit tests - if: inputs.run_unit_tests - uses: ./.github/actions/test - with: - log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}} - test_label_regexp: ${{ inputs.test_label_regexp }} - aws_key_id: ${{secrets.AWS_KEY_ID}} - aws_key_value: ${{secrets.AWS_KEY_VALUE}} - aws_bucket: ${{vars.AWS_BUCKET}} - aws_endpoint: ${{vars.AWS_ENDPOINT}} - testman_token: ${{secrets.TESTMO_TOKEN}} - testman_url: ${{vars.TESTMO_URL}} - testman_project_id: ${{vars.TESTMO_PROJECT_ID}} - - name: Run functional tests - if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure' - uses: ./.github/actions/test_python - with: - log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}} - test_label_regexp: ${{ inputs.test_label_regexp }} - aws_key_id: ${{secrets.AWS_KEY_ID}} - aws_key_value: ${{secrets.AWS_KEY_VALUE}} - aws_bucket: ${{vars.AWS_BUCKET}} - aws_endpoint: ${{vars.AWS_ENDPOINT}} - - release-runner: - name: Release self-hosted YC runner if provided on-demand - needs: - - provide-runner # required to get output from the start-runner job - - main # required to wait when the main job is done - runs-on: ubuntu-latest - if: always() - steps: - - name: Stop YC runner - uses: yc-actions/yc-github-runner@v1 - with: - mode: stop - yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }} - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.provide-runner.outputs.label }} - instance-id: ${{ needs.provide-runner.outputs.instance-id }} - diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml deleted file mode 100644 index fa726a115d..0000000000 --- a/.github/workflows/build_and_test_provisioned.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build-and-Test-Provisioned - -on: - workflow_call: - inputs: - runner_label: - required: true - type: string - run_build: - type: boolean - default: true - sanitizer: - required: false - type: string - run_unit_tests: - type: boolean - default: true - run_functional_tests: - type: boolean - default: false - test_label_regexp: - required: false - type: string - workflow_dispatch: - inputs: - runner_label: - required: true - type: string - run_build: - type: boolean - default: true - sanitizer: - required: false - type: string - run_unit_tests: - type: boolean - default: true - run_functional_tests: - type: boolean - default: true - test_label_regexp: - required: false - type: string - -jobs: - main: - name: Build and test - runs-on: [ self-hosted, Provisioned, "${{ inputs.runner_label }}" ] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Build - uses: ./.github/actions/build - if: inputs.run_build - with: - sanitizer: ${{ inputs.sanitizer }} - - name: Run unit tests - uses: ./.github/actions/test - if: inputs.run_unit_tests - with: - log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}} - test_label_regexp: ${{ inputs.test_label_regexp }} - aws_key_id: ${{secrets.AWS_KEY_ID}} - aws_key_value: ${{secrets.AWS_KEY_VALUE}} - aws_bucket: ${{vars.AWS_BUCKET}} - aws_endpoint: ${{vars.AWS_ENDPOINT}} - testman_token: ${{secrets.TESTMO_TOKEN}} - testman_url: ${{vars.TESTMO_URL}} - testman_project_id: ${{vars.TESTMO_PROJECT_ID}} - - name: Run functional tests - if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure' - uses: ./.github/actions/test_python - with: - log_suffix: ${{format('{0}{1}', inputs.runner_label, inputs.sanitizer)}} - test_label_regexp: ${{ inputs.test_label_regexp }} - aws_key_id: ${{secrets.AWS_KEY_ID}} - aws_key_value: ${{secrets.AWS_KEY_VALUE}} - aws_bucket: ${{vars.AWS_BUCKET}} - aws_endpoint: ${{vars.AWS_ENDPOINT}} - diff --git a/.github/workflows/nightly_run.yaml b/.github/workflows/nightly_run.yaml deleted file mode 100644 index 015b68cd05..0000000000 --- a/.github/workflows/nightly_run.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Nightly-run -on: - schedule: - - cron: "0 1 * * *" - workflow_dispatch: - inputs: - test_label_regexp: - required: false - type: string - -jobs: - build_ondemand: - name: Build/test X64 - strategy: - matrix: ${{ fromJSON(vars.NIGHTLY_ONDEMAND_MATRIX) }} - fail-fast: false - - uses: ./.github/workflows/build_and_test_ondemand.yml - with: - sanitizer: ${{matrix.sanitizer}} - test_label_regexp: ${{inputs.test_label_regexp}} - secrets: inherit - - build_provisioned: - name: Build/test ARM64 - strategy: - matrix: ${{ fromJSON(vars.NIGHTLY_PROVISIONED_MATRIX) }} - fail-fast: false - uses: ./.github/workflows/build_and_test_provisioned.yml - with: - runner_label: ARM64 - sanitizer: ${{matrix.sanitizer}} - test_label_regexp: ${{inputs.test_label_regexp}} - secrets: inherit diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml deleted file mode 100644 index c50690b1de..0000000000 --- a/.github/workflows/pr_check.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: PR check -on: - pull_request_target: - branches: - - 'main' - - 'stable-*' - paths-ignore: - - 'ydb/docs/**' - types: - - 'opened' - - 'synchronize' - - 'reopened' - - 'labeled' -jobs: - - check-running-allowed: - runs-on: ubuntu-latest - outputs: - result: ${{ steps.check-ownership-membership.outputs.result }} - steps: - - name: Check if running tests is allowed - id: check-ownership-membership - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - script: | - // This is used primarily in forks. Repository owner - // should be allowed to run anything. - const userLogin = context.payload.pull_request.user.login; - if (context.payload.repository.owner.login == userLogin) { - return true; - } - - const response = await github.rest.orgs.checkMembershipForUser({ - org: context.payload.organization.login, - username: userLogin, - }); - - // How to interpret membership status code: - // https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#check-organization-membership-for-a-user - if (response.status == '204') { - return true; - } - - const labels = context.payload.pull_request.labels; - const okToTestLabel = labels.find( - label => label.name == 'ok-to-test' - ); - return okToTestLabel !== undefined; - - name: comment-if-waiting-on-ok - if: steps.check-ownership-membership.outputs.result == 'false' && - github.event.action == 'opened' - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!' - }); - - name: cleanup-test-label - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - script: | - const { owner, repo } = context.repo; - const prNumber = context.payload.pull_request.number; - const labelToRemove = 'ok-to-test'; - try { - const result = await github.rest.issues.removeLabel({ - owner, - repo, - issue_number: prNumber, - name: labelToRemove - }); - } catch(e) { - // ignore the 404 error that arises - // when the label did not exist for the - // organization member - console.log(e); - } - build_and_test: - needs: - - check-running-allowed - if: needs.check-running-allowed.outputs.result == 'true' - uses: ./.github/workflows/build_and_test_ondemand.yml - with: - test_label_regexp: '(SMALL|MEDIUM)' - secrets: inherit diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml deleted file mode 100644 index dab4270299..0000000000 --- a/.github/workflows/prepare_vm_for_build.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Prepare VM for YDB build - -on: - workflow_call: - inputs: - runner_label: - required: true - type: string - secrets: - rc_auth: - required: false - workflow_dispatch: - inputs: - runner_label: - required: true - type: string - -jobs: - prepare: - runs-on: ${{ inputs.runner_label }} # run the job on a particular runner - steps: - - name: Install YDB Build dependencies - shell: bash - run: | - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add - - echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null - sudo apt-get update - sudo apt-get -y install python-is-python3 git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev parallel - sudo pip3 install conan==1.59 pytest==7.1.3 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow - - name: Install AllureCtl - if: false - shell: bash - run: | - wget https://github.com/allure-framework/allurectl/releases/latest/download/allurectl_linux_386 -O ~/allurectl - chmod +x ~/allurectl - - name: Checkout ccache - uses: actions/checkout@v3 - with: - repository: ccache/ccache - path: ccache - - name: Build ccache - shell: bash - run: | - pwd - cd ccache - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - make -j32 - make install - mkdir -p ~/.ccache - echo 'remote_storage = http://${{secrets.rc_auth}}${{vars.REMOTE_CACHE_URL}}' > /root/.ccache/ccache.conf - echo 'max_size = 50G' >> /root/.ccache/ccache.conf - ccache -p diff --git a/contrib/libs/grpc/CMakeLists.darwin.txt b/contrib/libs/grpc/CMakeLists.darwin.txt index 9bb0c69fdb..7023bec32e 100644 --- a/contrib/libs/grpc/CMakeLists.darwin.txt +++ b/contrib/libs/grpc/CMakeLists.darwin.txt @@ -641,10 +641,10 @@ target_sources(contrib-libs-grpc PRIVATE ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/tls/tls_credentials.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/tls/tls_utils.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/xds/xds_credentials.cc + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.cpp ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/alts/alts_security_connector.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/fake/fake_security_connector.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc - ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.cpp ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_system_roots_fallback.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_system_roots_linux.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/local/local_security_connector.cc diff --git a/contrib/libs/grpc/CMakeLists.linux-aarch64.txt b/contrib/libs/grpc/CMakeLists.linux-aarch64.txt index d4e3b955f5..4ac27dae52 100644 --- a/contrib/libs/grpc/CMakeLists.linux-aarch64.txt +++ b/contrib/libs/grpc/CMakeLists.linux-aarch64.txt @@ -642,10 +642,10 @@ target_sources(contrib-libs-grpc PRIVATE ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/tls/tls_credentials.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/tls/tls_utils.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/xds/xds_credentials.cc + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.cpp ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/alts/alts_security_connector.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/fake/fake_security_connector.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc - ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.cpp ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_system_roots_fallback.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_system_roots_linux.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/local/local_security_connector.cc diff --git a/contrib/libs/grpc/CMakeLists.linux.txt b/contrib/libs/grpc/CMakeLists.linux.txt index d4e3b955f5..4ac27dae52 100644 --- a/contrib/libs/grpc/CMakeLists.linux.txt +++ b/contrib/libs/grpc/CMakeLists.linux.txt @@ -642,10 +642,10 @@ target_sources(contrib-libs-grpc PRIVATE ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/tls/tls_credentials.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/tls/tls_utils.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/credentials/xds/xds_credentials.cc + ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.cpp ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/alts/alts_security_connector.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/fake/fake_security_connector.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc - ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.cpp ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_system_roots_fallback.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/load_system_roots_linux.cc ${CMAKE_SOURCE_DIR}/contrib/libs/grpc/src/core/lib/security/security_connector/local/local_security_connector.cc diff --git a/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.cpp b/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.cpp new file mode 100644 index 0000000000..15629cfbe7 --- /dev/null +++ b/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.cpp @@ -0,0 +1,16 @@ +#include "add_arcadia_root_certs.h" +#include "grpc/support/alloc.h" + +#include <library/cpp/resource/resource.h> + +namespace grpc_core { + grpc_slice AddArcadiaRootCerts(grpc_slice systemCerts) { + TString cacert = NResource::Find("/builtin/cacert"); + size_t sumSize = cacert.size() + GRPC_SLICE_LENGTH(systemCerts); + char* bundleString = static_cast<char*>(gpr_zalloc(sumSize + 1)); // With \0. + memcpy(bundleString, cacert.data(), cacert.size()); + memcpy(bundleString + cacert.size(), GRPC_SLICE_START_PTR(systemCerts), GRPC_SLICE_LENGTH(systemCerts)); + grpc_slice_unref(systemCerts); + return grpc_slice_new(bundleString, sumSize, gpr_free); + } +} diff --git a/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.h b/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.h index fb57648e48..d41bb2032f 100644 --- a/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.h +++ b/contrib/libs/grpc/src/core/lib/security/security_connector/add_arcadia_root_certs.h @@ -3,5 +3,5 @@ #include <grpc/slice.h> namespace grpc_core { - grpc_slice LoadArcadiaRootCerts(); + grpc_slice AddArcadiaRootCerts(grpc_slice systemCerts); } diff --git a/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.cpp b/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.cpp deleted file mode 100644 index d07d3b21b8..0000000000 --- a/contrib/libs/grpc/src/core/lib/security/security_connector/load_arcadia_root_certs.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "load_arcadia_root_certs.h" - -#include <library/cpp/resource/resource.h> - -namespace grpc_core { - grpc_slice LoadArcadiaRootCerts() { - TString cacert = NResource::Find("/builtin/cacert"); - return grpc_slice_from_copied_buffer(cacert.data(), cacert.size() + 1); // With \0. - } -} diff --git a/contrib/libs/grpc/src/core/lib/security/security_connector/ssl_utils.cc b/contrib/libs/grpc/src/core/lib/security/security_connector/ssl_utils.cc index d7e5ba282e..c19641fcc5 100644 --- a/contrib/libs/grpc/src/core/lib/security/security_connector/ssl_utils.cc +++ b/contrib/libs/grpc/src/core/lib/security/security_connector/ssl_utils.cc @@ -39,7 +39,7 @@ #include "src/core/lib/security/security_connector/ssl_utils_config.h" #include "src/core/tsi/ssl_transport_security.h" -#include "load_arcadia_root_certs.h" +#include "add_arcadia_root_certs.h" /* -- Constants. -- */ @@ -581,13 +581,11 @@ grpc_slice DefaultSslRootStore::ComputePemRootCerts() { } gpr_free(pem_root_certs); } - // Load Arcadia certs. - if (GRPC_SLICE_IS_EMPTY(result)) { - result = LoadArcadiaRootCerts(); - } // Try loading roots from OS trust store if flag is enabled. if (GRPC_SLICE_IS_EMPTY(result) && !not_use_system_roots) { result = LoadSystemRootCerts(); + // Merge with Arcadia certs. + result = AddArcadiaRootCerts(result); } // Fallback to roots manually shipped with gRPC. if (GRPC_SLICE_IS_EMPTY(result) && diff --git a/contrib/libs/libfyaml/include/libfyaml.h b/contrib/libs/libfyaml/include/libfyaml.h index 2d8bed3ba6..263e7d0dc8 100644 --- a/contrib/libs/libfyaml/include/libfyaml.h +++ b/contrib/libs/libfyaml/include/libfyaml.h @@ -2772,6 +2772,20 @@ fy_node_get_style(struct fy_node *fyn) FY_EXPORT; /** + * fy_node_set_style() - Set the node style + * + * Set the node rendering style. + * If current node style is alias it won't be changed + * to save document structure + * + * @fyn: The node + * @style: The node style + */ +void +fy_node_set_style(struct fy_node *fyn, enum fy_node_style style) + FY_EXPORT; + +/** * fy_node_is_scalar() - Check whether the node is a scalar * * Convenience method for checking whether a node is a scalar. diff --git a/contrib/libs/libfyaml/src/lib/fy-doc.c b/contrib/libs/libfyaml/src/lib/fy-doc.c index a40694a24c..436a0b1cca 100644 --- a/contrib/libs/libfyaml/src/lib/fy-doc.c +++ b/contrib/libs/libfyaml/src/lib/fy-doc.c @@ -3352,6 +3352,18 @@ enum fy_node_style fy_node_get_style(struct fy_node *fyn) return fyn ? fyn->style : FYNS_PLAIN; } +void fy_node_set_style(struct fy_node *fyn, enum fy_node_style style) +{ + if (!fyn) + return; + + /* ignore alias nodes to save document structure */ + if (fyn->style == FYNS_ALIAS) + return; + + fyn->style = style; +} + bool fy_node_is_null(struct fy_node *fyn) { if (!fyn) diff --git a/library/cpp/actors/core/monotonic.cpp b/library/cpp/actors/core/monotonic.cpp index 4d2c28ef0c..581d80d849 100644 --- a/library/cpp/actors/core/monotonic.cpp +++ b/library/cpp/actors/core/monotonic.cpp @@ -1,9 +1 @@ #include "monotonic.h" - -namespace NActors { - -ui64 GetMonotonicMicroSeconds() { - return NMonotonic::GetMonotonicMicroSeconds(); -} - -} // namespace NActors diff --git a/library/cpp/actors/core/monotonic.h b/library/cpp/actors/core/monotonic.h index 46fd13f335..2c53785390 100644 --- a/library/cpp/actors/core/monotonic.h +++ b/library/cpp/actors/core/monotonic.h @@ -5,7 +5,8 @@ namespace NActors { -ui64 GetMonotonicMicroSeconds(); +using NMonotonic::GetMonotonicMicroSeconds; using TMonotonic = NMonotonic::TMonotonic; + } // namespace NActors diff --git a/library/cpp/actors/core/monotonic_provider.cpp b/library/cpp/actors/core/monotonic_provider.cpp index 9e5bb1aadf..f8d91a4eec 100644 --- a/library/cpp/actors/core/monotonic_provider.cpp +++ b/library/cpp/actors/core/monotonic_provider.cpp @@ -1,9 +1 @@ #include "monotonic_provider.h" - -namespace NActors { - -TIntrusivePtr<NActors::IMonotonicTimeProvider> CreateDefaultMonotonicTimeProvider() { - return NMonotonic::CreateDefaultMonotonicTimeProvider(); -} - -} // namespace NActors diff --git a/library/cpp/actors/core/monotonic_provider.h b/library/cpp/actors/core/monotonic_provider.h index 1f6c44aeb9..befe4f7b90 100644 --- a/library/cpp/actors/core/monotonic_provider.h +++ b/library/cpp/actors/core/monotonic_provider.h @@ -4,8 +4,8 @@ namespace NActors { -using IMonotonicTimeProvider = ::IMonotonicTimeProvider; +using IMonotonicTimeProvider = NMonotonic::IMonotonicTimeProvider; -TIntrusivePtr<IMonotonicTimeProvider> CreateDefaultMonotonicTimeProvider(); +using NMonotonic::CreateDefaultMonotonicTimeProvider; } // namespace NActors diff --git a/library/cpp/lwtrace/control.cpp b/library/cpp/lwtrace/control.cpp index d9404ff269..b0d40f7063 100644 --- a/library/cpp/lwtrace/control.cpp +++ b/library/cpp/lwtrace/control.cpp @@ -73,11 +73,18 @@ TTraceDeserializeStatus TManager::HandleTraceResponse( LWTRACK(DeserializationError, orbit, probe->Event.Name, probe->Event.GetProvider()); result.AddFailedEventName(v.GetName()); } else { + // in case of fork join probes would be like "start 0 fork 1 ....... join 10 forked 5 forked 6" ui64 timestamp = EpochNanosecondsToCycles(v.GetTimestampNanosec()); + if (timestamp > prev) { + timestamp = prev + (timestamp-prev)*timeScale + timeOffset; + } else { + timestamp += timeOffset; + } + orbit.AddProbe( probe, params, - prev + (timestamp-prev)*timeScale + timeOffset); + timestamp); probe->Event.Signature.DestroyParams(params); prev = timestamp; } diff --git a/library/cpp/lwtrace/trace_ut.cpp b/library/cpp/lwtrace/trace_ut.cpp index afb2f4e53f..9a29923e28 100644 --- a/library/cpp/lwtrace/trace_ut.cpp +++ b/library/cpp/lwtrace/trace_ut.cpp @@ -717,27 +717,59 @@ Y_UNIT_TEST_SUITE(LWTraceTrace) { TManager manager(*Singleton<TProbeRegistry>(), false); TOrbit orbit; + TOrbit child; + TTraceRequest req; req.SetIsTraced(true); - manager.HandleTraceRequest(req, orbit); + bool traced = manager.HandleTraceRequest(req, orbit); + UNIT_ASSERT(traced); LWTRACK(NoParam, orbit); + + orbit.Fork(child); + LWTRACK(IntParam, orbit, 1); - LWTRACK(StringParam, orbit, "str"); + LWTRACK(IntParam, child, 2); + + LWTRACK(StringParam, orbit, "str1"); + LWTRACK(StringParam, child, "str2"); + LWTRACK(EnumParams, orbit, ValueA, EEnumClass::ValueC); LWTRACK(InstantParam, orbit, TInstant::Seconds(42)); LWTRACK(DurationParam, orbit, TDuration::MilliSeconds(146)); LWTRACK(ProtoEnum, orbit, OT_EQ); LWTRACK(IntIntParams, orbit, 1, 2); - TTraceResponse resp; - auto& r = *resp.MutableTrace(); - orbit.Serialize(0, r); + orbit.Join(child); + + TTraceResponse resp1; + auto& r1 = *resp1.MutableTrace(); + orbit.Serialize(0, r1); + + UNIT_ASSERT_VALUES_EQUAL(r1.EventsSize(), 12); + + TOrbit other; + traced = manager.HandleTraceRequest(req, other); + UNIT_ASSERT(traced); - TOrbit orbit1; UNIT_ASSERT_VALUES_EQUAL( - manager.HandleTraceResponse(resp, manager.GetProbesMap(), orbit1).IsSuccess, + manager.HandleTraceResponse(resp1, manager.GetProbesMap(), other).IsSuccess, true); + + TTraceResponse resp2; + auto& r2 = *resp2.MutableTrace(); + other.Serialize(0, r2); + UNIT_ASSERT_VALUES_EQUAL(r2.EventsSize(), 12); + + TString proto1; + bool parsed = NProtoBuf::TextFormat::PrintToString(resp1, &proto1); + UNIT_ASSERT(parsed); + + TString proto2; + parsed = NProtoBuf::TextFormat::PrintToString(resp2, &proto2); + UNIT_ASSERT(parsed); + + UNIT_ASSERT_VALUES_EQUAL(proto1, proto2); } Y_UNIT_TEST(TrackForkJoin) { diff --git a/library/cpp/time_provider/monotonic.cpp b/library/cpp/time_provider/monotonic.cpp index 99126080e2..f9fb6c5e19 100644 --- a/library/cpp/time_provider/monotonic.cpp +++ b/library/cpp/time_provider/monotonic.cpp @@ -1,28 +1,81 @@ #include "monotonic.h" #include <chrono> +#include <optional> +#include <system_error> +#include <util/system/platform.h> + +#ifdef _linux_ + #include <time.h> + #include <string.h> +#endif namespace NMonotonic { -namespace { -// Unfortunately time_since_epoch() is sometimes negative on wine -// Remember initial time point at program start and use offsets from that -std::chrono::steady_clock::time_point MonotonicOffset = std::chrono::steady_clock::now(); -} + namespace { +#if defined(_linux_) && defined(CLOCK_BOOTTIME) + std::optional<ui64> GetClockBootTimeMicroSeconds() { + struct timespec t; + std::optional<ui64> r; + if (0 == ::clock_gettime(CLOCK_BOOTTIME, &t)) { + r.emplace(t.tv_nsec / 1000ULL + t.tv_sec * 1000000ULL); + } + return r; + } +#endif + + struct TMonotonicSupport { +#if defined(_linux_) && defined(CLOCK_BOOTTIME) + // We remember initial offset to measure time relative to program + // start and so we never return a zero time. + std::optional<ui64> BootTimeOffset; +#endif + // Unfortunately time_since_epoch() is sometimes negative on wine + // Remember initial time point at program start and use offsets from that + std::chrono::steady_clock::time_point SteadyClockOffset; + + TMonotonicSupport() { +#if defined(_linux_) && defined(CLOCK_BOOTTIME) + BootTimeOffset = GetClockBootTimeMicroSeconds(); +#endif + SteadyClockOffset = std::chrono::steady_clock::now(); + } + + ui64 GetMicroSeconds() const { +#if defined(_linux_) && defined(CLOCK_BOOTTIME) + if (Y_LIKELY(BootTimeOffset)) { + auto r = GetClockBootTimeMicroSeconds(); + if (Y_UNLIKELY(!r)) { + throw std::system_error( + std::error_code(errno, std::system_category()), + "clock_gettime(CLOCK_BOOTTIME) failed"); + } + // Note: we add 1 so we never return zero + return *r - *BootTimeOffset + 1; + } +#endif + auto elapsed = std::chrono::steady_clock::now() - SteadyClockOffset; + auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(elapsed).count(); + // Steady clock is supposed to never jump backwards, but it's + // better to be safe in case of buggy implementations + if (Y_UNLIKELY(microseconds < 0)) { + microseconds = 0; + } + // Note: we add 1 so we never return zero + return ui64(microseconds) + 1; + } + }; -ui64 GetMonotonicMicroSeconds() { - auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - MonotonicOffset).count(); - // Steady clock is supposed to never jump backwards, but it's better to be safe in case of buggy implementations - if (Y_UNLIKELY(microseconds < 0)) { - microseconds = 0; + TMonotonicSupport MonotonicSupport; + } + + ui64 GetMonotonicMicroSeconds() { + return MonotonicSupport.GetMicroSeconds(); } - // Add one so we never return zero - return microseconds + 1; -} -} // namespace TMonotonic +} -template<> +template <> void Out<NMonotonic::TMonotonic>( IOutputStream& o, NMonotonic::TMonotonic t) diff --git a/library/cpp/time_provider/monotonic.h b/library/cpp/time_provider/monotonic.h index e36902d884..a1258e3342 100644 --- a/library/cpp/time_provider/monotonic.h +++ b/library/cpp/time_provider/monotonic.h @@ -1,111 +1,124 @@ #pragma once #include <util/datetime/base.h> -namespace NMonotonic { -/** - * Returns current monotonic time in microseconds - */ -ui64 GetMonotonicMicroSeconds(); - -/** - * Similar to TInstant, but measuring monotonic time - */ -class TMonotonic: public TTimeBase<TMonotonic> { - using TBase = TTimeBase<TMonotonic>; - -private: - constexpr explicit TMonotonic(TValue value) noexcept - : TBase(value) { - } - -public: - constexpr TMonotonic() noexcept { - } - - static constexpr TMonotonic FromValue(TValue value) noexcept { - return TMonotonic(value); - } - - static inline TMonotonic Now() { - return TMonotonic::MicroSeconds(GetMonotonicMicroSeconds()); - } - - using TBase::Days; - using TBase::Hours; - using TBase::MicroSeconds; - using TBase::MilliSeconds; - using TBase::Minutes; - using TBase::Seconds; - static constexpr TMonotonic Max() noexcept { - return TMonotonic(::Max<ui64>()); - } - - static constexpr TMonotonic Zero() noexcept { - return TMonotonic(); - } - - static constexpr TMonotonic MicroSeconds(ui64 us) noexcept { - return TMonotonic(TInstant::MicroSeconds(us).GetValue()); - } - - static constexpr TMonotonic MilliSeconds(ui64 ms) noexcept { - return TMonotonic(TInstant::MilliSeconds(ms).GetValue()); - } - - static constexpr TMonotonic Seconds(ui64 s) noexcept { - return TMonotonic(TInstant::Seconds(s).GetValue()); - } - - static constexpr TMonotonic Minutes(ui64 m) noexcept { - return TMonotonic(TInstant::Minutes(m).GetValue()); - } - - static constexpr TMonotonic Hours(ui64 h) noexcept { - return TMonotonic(TInstant::Hours(h).GetValue()); - } - - static constexpr TMonotonic Days(ui64 d) noexcept { - return TMonotonic(TInstant::Days(d).GetValue()); - } +namespace NMonotonic { - template<class T> - inline TMonotonic& operator+=(const T& t) noexcept { - return (*this = (*this + t)); - } + /** + * Returns current monotonic time in microseconds + * + * On Linux uses CLOCK_BOOTTIME under the hood, so it includes time passed + * during suspend and makes it safe for measuring lease times. + */ + ui64 GetMonotonicMicroSeconds(); + + /** + * Similar to TInstant, but measuring monotonic time + * + * On Linux uses CLOCK_BOOTTIME under the hood, so it includes time passed + * during suspend and makes it safe for measuring lease times. + */ + class TMonotonic: public TTimeBase<TMonotonic> { + using TBase = TTimeBase<TMonotonic>; + + protected: + constexpr explicit TMonotonic(TValue value) noexcept + : TBase(value) + { + } + + public: + constexpr TMonotonic() noexcept { + } + + static constexpr TMonotonic FromValue(TValue value) noexcept { + return TMonotonic(value); + } + + static inline TMonotonic Now() { + return TMonotonic::MicroSeconds(GetMonotonicMicroSeconds()); + } + + using TBase::Days; + using TBase::Hours; + using TBase::MicroSeconds; + using TBase::MilliSeconds; + using TBase::Minutes; + using TBase::Seconds; + + static constexpr TMonotonic Max() noexcept { + return TMonotonic::FromValue(::Max<ui64>()); + } + + static constexpr TMonotonic Zero() noexcept { + return TMonotonic::FromValue(0); + } + + static constexpr TMonotonic MicroSeconds(ui64 us) noexcept { + return TMonotonic::FromValue(TInstant::MicroSeconds(us).GetValue()); + } + + static constexpr TMonotonic MilliSeconds(ui64 ms) noexcept { + return TMonotonic::FromValue(TInstant::MilliSeconds(ms).GetValue()); + } + + static constexpr TMonotonic Seconds(ui64 s) noexcept { + return TMonotonic::FromValue(TInstant::Seconds(s).GetValue()); + } + + static constexpr TMonotonic Minutes(ui64 m) noexcept { + return TMonotonic::FromValue(TInstant::Minutes(m).GetValue()); + } + + static constexpr TMonotonic Hours(ui64 h) noexcept { + return TMonotonic::FromValue(TInstant::Hours(h).GetValue()); + } + + static constexpr TMonotonic Days(ui64 d) noexcept { + return TMonotonic::FromValue(TInstant::Days(d).GetValue()); + } + + template <class T> + inline TMonotonic& operator+=(const T& t) noexcept { + return (*this = (*this + t)); + } + + template <class T> + inline TMonotonic& operator-=(const T& t) noexcept { + return (*this = (*this - t)); + } + }; - template<class T> - inline TMonotonic& operator-=(const T& t) noexcept { - return (*this = (*this - t)); - } -}; } // namespace NMonotonic Y_DECLARE_PODTYPE(NMonotonic::TMonotonic); -template<> -struct THash<NMonotonic::TMonotonic> { - size_t operator()(const NMonotonic::TMonotonic& key) const { - return THash<NMonotonic::TMonotonic::TValue>()(key.GetValue()); - } -}; +namespace std { + template <> + struct hash<NMonotonic::TMonotonic> { + size_t operator()(const NMonotonic::TMonotonic& key) const noexcept { + return hash<NMonotonic::TMonotonic::TValue>()(key.GetValue()); + } + }; +} namespace NMonotonic { -constexpr TDuration operator-(const TMonotonic& l, const TMonotonic& r) { - return TInstant::FromValue(l.GetValue()) - TInstant::FromValue(r.GetValue()); -} + constexpr TDuration operator-(const TMonotonic& l, const TMonotonic& r) { + return TInstant::FromValue(l.GetValue()) - TInstant::FromValue(r.GetValue()); + } -constexpr TMonotonic operator+(const TMonotonic& l, const TDuration& r) { - TInstant result = TInstant::FromValue(l.GetValue()) + r; - return TMonotonic::FromValue(result.GetValue()); -} + constexpr TMonotonic operator+(const TMonotonic& l, const TDuration& r) { + TInstant result = TInstant::FromValue(l.GetValue()) + r; + return TMonotonic::FromValue(result.GetValue()); + } -constexpr TMonotonic operator-(const TMonotonic& l, const TDuration& r) { - TInstant result = TInstant::FromValue(l.GetValue()) - r; - return TMonotonic::FromValue(result.GetValue()); -} + constexpr TMonotonic operator-(const TMonotonic& l, const TDuration& r) { + TInstant result = TInstant::FromValue(l.GetValue()) - r; + return TMonotonic::FromValue(result.GetValue()); + } } // namespace NMonotonic +// TODO: remove, alias for compatibility using TMonotonic = NMonotonic::TMonotonic; diff --git a/library/cpp/time_provider/monotonic_provider.cpp b/library/cpp/time_provider/monotonic_provider.cpp index 22937bd873..6b116a0e73 100644 --- a/library/cpp/time_provider/monotonic_provider.cpp +++ b/library/cpp/time_provider/monotonic_provider.cpp @@ -1,32 +1,32 @@ #include "monotonic_provider.h" -namespace { -TIntrusivePtr<IMonotonicTimeProvider> GlobalMonotonicTimeProvider; -} - -void TMonotonicOperator::RegisterProvider(TIntrusivePtr<IMonotonicTimeProvider> provider) { - GlobalMonotonicTimeProvider = provider; -} +namespace NMonotonic { -NMonotonic::TMonotonic TMonotonicOperator::Now() { - if (GlobalMonotonicTimeProvider) { - return GlobalMonotonicTimeProvider->Now(); - } else { - return TMonotonic::Now(); + namespace { + TIntrusivePtr<IMonotonicTimeProvider> GlobalMonotonicTimeProvider; } -} -namespace NMonotonic { + void TMonotonicOperator::RegisterProvider(TIntrusivePtr<IMonotonicTimeProvider> provider) { + GlobalMonotonicTimeProvider = provider; + } -class TDefaultMonotonicTimeProvider: public IMonotonicTimeProvider { -public: - TMonotonic Now() override { - return TMonotonic::Now(); + NMonotonic::TMonotonic TMonotonicOperator::Now() { + if (GlobalMonotonicTimeProvider) { + return GlobalMonotonicTimeProvider->Now(); + } else { + return TMonotonic::Now(); + } } -}; -TIntrusivePtr<IMonotonicTimeProvider> CreateDefaultMonotonicTimeProvider() { - return TIntrusivePtr<IMonotonicTimeProvider>(new TDefaultMonotonicTimeProvider); -} + class TDefaultMonotonicTimeProvider: public IMonotonicTimeProvider { + public: + TMonotonic Now() override { + return TMonotonic::Now(); + } + }; + + TIntrusivePtr<IMonotonicTimeProvider> CreateDefaultMonotonicTimeProvider() { + return TIntrusivePtr<IMonotonicTimeProvider>(new TDefaultMonotonicTimeProvider); + } } diff --git a/library/cpp/time_provider/monotonic_provider.h b/library/cpp/time_provider/monotonic_provider.h index 966e2e496b..17b57e4075 100644 --- a/library/cpp/time_provider/monotonic_provider.h +++ b/library/cpp/time_provider/monotonic_provider.h @@ -3,19 +3,25 @@ #include <util/datetime/base.h> #include "monotonic.h" -class IMonotonicTimeProvider: public TThrRefBase { -public: - virtual TMonotonic Now() = 0; -}; +namespace NMonotonic { -class TMonotonicOperator { -public: - static void RegisterProvider(TIntrusivePtr<IMonotonicTimeProvider> provider); - static TMonotonic Now(); -}; + class IMonotonicTimeProvider: public TThrRefBase { + public: + virtual TMonotonic Now() = 0; + }; -namespace NMonotonic { + class TMonotonicOperator { + public: + static void RegisterProvider(TIntrusivePtr<IMonotonicTimeProvider> provider); + static TMonotonic Now(); + }; -TIntrusivePtr<IMonotonicTimeProvider> CreateDefaultMonotonicTimeProvider(); + TIntrusivePtr<IMonotonicTimeProvider> CreateDefaultMonotonicTimeProvider(); } + +// TODO: remove, alias for compatibility +using IMonotonicTimeProvider = NMonotonic::IMonotonicTimeProvider; + +// TODO: remove, alias for compatibility +using NMonotonic::CreateDefaultMonotonicTimeProvider; diff --git a/library/cpp/time_provider/time_provider.cpp b/library/cpp/time_provider/time_provider.cpp index 687681f1ff..e1045dae05 100644 --- a/library/cpp/time_provider/time_provider.cpp +++ b/library/cpp/time_provider/time_provider.cpp @@ -30,7 +30,7 @@ TIntrusivePtr<ITimeProvider> CreateDeterministicTimeProvider(ui64 seed) { } namespace { -TIntrusivePtr<ITimeProvider> GlobalTimeProvider; + TIntrusivePtr<ITimeProvider> GlobalTimeProvider; } void TInstantOperator::RegisterProvider(TIntrusivePtr<ITimeProvider> provider) { diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp index b8d8b17596..d006001b9e 100644 --- a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp +++ b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp @@ -155,18 +155,6 @@ enum EEmitterCfgFlags { Default = FYECF_DEFAULT, }; -enum class ENodeStyle { - Any = FYNS_ANY, - Flow = FYNS_FLOW, - Block = FYNS_BLOCK, - Plain = FYNS_PLAIN, - SingleQuoted = FYNS_SINGLE_QUOTED, - DoubleQuoted = FYNS_DOUBLE_QUOTED, - Literal = FYNS_LITERAL, - Folded = FYNS_FOLDED, - Alias = FYNS_ALIAS, -}; - enum class ENodeWalkFlags { DontFollow = FYNWF_DONT_FOLLOW, Follow = FYNWF_FOLLOW, @@ -330,6 +318,115 @@ TString TNodeRef::Scalar() const { return TString(text, size); } +TMark TNodeRef::BeginMark() const { + ENSURE_NODE_NOT_EMPTY(Node_); + std::unique_ptr<fy_document_iterator, void(*)(fy_document_iterator*)> it( + fy_document_iterator_create(), + &fy_document_iterator_destroy); + fy_document_iterator_node_start(it.get(), Node_); + auto deleter = [&](fy_event* fye){ fy_document_iterator_event_free(it.get(), fye); }; + std::unique_ptr<fy_event, decltype(deleter)> ev( + fy_document_iterator_body_next(it.get()), + deleter); + auto* mark = fy_event_start_mark(ev.get()); + + if (!mark) { + ythrow yexception() << "can't get begin mark for a node"; + } + + return TMark{ + mark->input_pos, + mark->line, + mark->column, + }; +} + +bool IsComplexType(ENodeType type) { + return type == ENodeType::Mapping || type == ENodeType::Sequence; +} + +fy_event_type GetOpenEventType(ENodeType type) { + switch(type) { + case ENodeType::Mapping: + return FYET_MAPPING_START; + case ENodeType::Sequence: + return FYET_SEQUENCE_START; + default: + Y_FAIL("Not a brackets type"); + } +} + +fy_event_type GetCloseEventType(ENodeType type) { + switch(type) { + case ENodeType::Mapping: + return FYET_MAPPING_END; + case ENodeType::Sequence: + return FYET_SEQUENCE_END; + default: + Y_FAIL("Not a brackets type"); + } +} + +TMark TNodeRef::EndMark() const { + ENSURE_NODE_NOT_EMPTY(Node_); + std::unique_ptr<fy_document_iterator, void(*)(fy_document_iterator*)> it( + fy_document_iterator_create(), + &fy_document_iterator_destroy); + fy_document_iterator_node_start(it.get(), Node_); + + auto deleter = [&](fy_event* fye){ fy_document_iterator_event_free(it.get(), fye); }; + std::unique_ptr<fy_event, decltype(deleter)> prevEv( + nullptr, + deleter); + std::unique_ptr<fy_event, decltype(deleter)> ev( + fy_document_iterator_body_next(it.get()), + deleter); + + if (IsComplexType(Type())) { + int openBrackets = 0; + if (ev->type == GetOpenEventType(Type())) { + ++openBrackets; + } + if (ev->type == GetCloseEventType(Type())) { + --openBrackets; + } + while (ev->type != GetCloseEventType(Type()) || openBrackets != 0) { + std::unique_ptr<fy_event, decltype(deleter)> cur( + fy_document_iterator_body_next(it.get()), + deleter); + if (cur == nullptr) { + break; + } + if (cur->type == GetOpenEventType(Type())) { + ++openBrackets; + } + if (cur->type == GetCloseEventType(Type())) { + --openBrackets; + } + if (fy_event_get_node_style(cur.get()) != FYNS_BLOCK) { + prevEv.reset(ev.release()); + ev.reset(cur.release()); + } + } + } + + auto* mark = fy_event_end_mark(ev.get()); + + if (!mark && prevEv) { + mark = fy_event_end_mark(prevEv.get()); + } + + if (!mark) { + ythrow yexception() << "can't get end mark for a node"; + } + + return TMark{ + mark->input_pos, + mark->line, + mark->column, + }; +} + TMapping TNodeRef::Map() const { ENSURE_NODE_NOT_EMPTY(Node_); Y_ENSURE_EX(fy_node_is_mapping(Node_), TFyamlEx() << "Node is not Mapping: " << Path()); @@ -400,6 +497,16 @@ std::unique_ptr<char, void(*)(char*)> TNodeRef::EmitToCharArray() const { return res; } +void TNodeRef::SetStyle(ENodeStyle style) { + ENSURE_NODE_NOT_EMPTY(Node_); + fy_node_set_style(Node_, (enum fy_node_style)style); +} + +ENodeStyle TNodeRef::Style() const { + ENSURE_NODE_NOT_EMPTY(Node_); + return (ENodeStyle)fy_node_get_style(Node_); +} + void TNodeRef::SetUserData(NDetail::IBasicUserData* data) { ENSURE_NODE_NOT_EMPTY(Node_); fy_node_set_meta(Node_, data); @@ -429,6 +536,12 @@ TNodeRef TNodePairRef::Key() const { return TNodeRef(fy_node_pair_key(Pair_)); } +int TNodePairRef::Index(const TNodeRef& node) const { + ENSURE_NODE_NOT_EMPTY(node); + ENSURE_NODE_NOT_EMPTY(Pair_); + return fy_node_mapping_get_pair_index(node.Node_, Pair_); +} + void TNodePairRef::SetKey(const TNodeRef& node) { ENSURE_NODE_NOT_EMPTY(Pair_); ENSURE_NODE_NOT_EMPTY(node); @@ -542,6 +655,9 @@ void TMapping::Remove(const TNodePairRef& toRemove) { ENSURE_NODE_NOT_EMPTY(Node_); ENSURE_NODE_NOT_EMPTY(toRemove); NDetail::RethrowOnError(fy_node_mapping_remove(Node_, toRemove.Pair_), Node_); + fy_node_free(fy_node_pair_key(toRemove.Pair_)); + fy_node_free(fy_node_pair_value(toRemove.Pair_)); + free(toRemove.Pair_); } TMappingIterator TMapping::Remove(const TMappingIterator& toRemove) { @@ -819,7 +935,7 @@ std::unique_ptr<char, void(*)(char*)> TDocument::EmitToCharArray() const { std::unique_ptr<char, void(*)(char*)> res( fy_emit_document_to_string( Document_.get(), - (fy_emitter_cfg_flags)(FYECF_DEFAULT | FYECF_OUTPUT_COMMENTS)), &NDetail::FreeChar); + (fy_emitter_cfg_flags)(FYECF_DEFAULT | FYECF_MODE_PRETTY | FYECF_OUTPUT_COMMENTS)), &NDetail::FreeChar); return res; } diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.h b/library/cpp/yaml/fyamlcpp/fyamlcpp.h index bcf9362d73..6df7b244c6 100644 --- a/library/cpp/yaml/fyamlcpp/fyamlcpp.h +++ b/library/cpp/yaml/fyamlcpp/fyamlcpp.h @@ -15,6 +15,7 @@ struct fy_document; struct fy_diag; struct fy_document_iterator; struct fy_node_pair; +extern "C" struct fy_node *fy_node_buildf(struct fy_document *fyd, const char *fmt, ...); namespace NFyaml { @@ -32,6 +33,18 @@ enum class ENodeType { Mapping, }; +enum class ENodeStyle { + Any = -1, + Flow, + Block, + Plain, + SingleQuoted, + DoubleQuoted, + Literal, + Folded, + Alias, +}; + namespace NDetail { class IBasicUserData { @@ -137,6 +150,10 @@ public: TString Scalar() const; + TMark BeginMark() const; + + TMark EndMark() const; + void Insert(const TNodeRef& node); bool Empty() const { return Node_ == nullptr; } @@ -155,6 +172,10 @@ public: std::unique_ptr<char, void(*)(char*)> EmitToCharArray() const; + void SetStyle(ENodeStyle style); + + ENodeStyle Style() const; + protected: fy_node* Node_ = nullptr; @@ -198,6 +219,8 @@ public: TNodeRef Key() const; + int Index(const TNodeRef& node) const; + void SetKey(const TNodeRef& node); TNodeRef Value() const; diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp b/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp index ffe1d4368c..2f6e14138c 100644 --- a/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp +++ b/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp @@ -11,6 +11,16 @@ Y_UNIT_TEST_SUITE(FYamlCpp) { UNIT_ASSERT_EQUAL((int)NFyaml::ENodeType::Scalar, FYNT_SCALAR); UNIT_ASSERT_EQUAL((int)NFyaml::ENodeType::Sequence, FYNT_SEQUENCE); UNIT_ASSERT_EQUAL((int)NFyaml::ENodeType::Mapping, FYNT_MAPPING); + + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Any, FYNS_ANY); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Flow, FYNS_FLOW); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Block, FYNS_BLOCK); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Plain, FYNS_PLAIN); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::SingleQuoted, FYNS_SINGLE_QUOTED); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::DoubleQuoted, FYNS_DOUBLE_QUOTED); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Literal, FYNS_LITERAL); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Folded, FYNS_FOLDED); + UNIT_ASSERT_EQUAL((int)NFyaml::ENodeStyle::Alias, FYNS_ALIAS); } Y_UNIT_TEST(ErrorHandling) { @@ -153,4 +163,365 @@ x: b UNIT_ASSERT_VALUES_EQUAL(seq[1].Map().at("b").Sequence().at(1).Scalar(), "2"); UNIT_ASSERT_VALUES_EQUAL(seq[1].Map().at("b").Sequence().at(2).Scalar(), "3"); } + + Y_UNIT_TEST(SimpleScalarMark) { + auto check = [](const TString& str, const NFyaml::TNodeRef& node) { + auto pos = str.find("123"); + auto endPos = pos + strlen("123"); + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + UNIT_ASSERT_VALUES_EQUAL(begin, pos); + UNIT_ASSERT_VALUES_EQUAL(end, endPos); + }; + + { + TString str = R"(obj: 123)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"(obj: 123 # test)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +# test +obj: 123 # test +# test +)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +--- +obj: 123 +... +)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +a: foo +test: [{obj: 123}] +b: bar + )"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("test").Sequence().at(0).Map().at("obj"); + check(str, node); + } + + { + TString str = R"(obj: '123')"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"(obj: '123' # test)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +# test +obj: '123' # test +# test +)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +--- +obj: '123' +... +)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +a: foo +test: [{obj: "123"}] +b: bar + )"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("test").Sequence().at(0).Map().at("obj"); + check(str, node); + } + + { + TString str = R"(obj: "123")"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"(obj: "123" # test)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +# test +obj: "123" # test +# test +)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +--- +obj: "123" +... +)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +a: foo +test: [{obj: "123"}] +b: bar + )"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("test").Sequence().at(0).Map().at("obj"); + check(str, node); + } + + { + TString str = R"( +a: foo +test: [{obj: !!int "123"}] +b: bar + )"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("test").Sequence().at(0).Map().at("obj"); + check(str, node); + } + + } + + Y_UNIT_TEST(MultilineScalarMark) { + { + TString str = R"(obj: >+2 + some + multiline + + scalar with couple words)"; + auto doc = NFyaml::TDocument::Parse(str); + auto node = doc.Root().Map().at("obj"); + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + UNIT_ASSERT_VALUES_EQUAL(begin, 9); + UNIT_ASSERT_VALUES_EQUAL(end, 55); + } + } + + Y_UNIT_TEST(MapMark) { + { + TString str = R"( +a: foo +map: !!map + internal_map1: {} + internal_map2: + internal_map3: + internal_map4: + value: 1 + internal_map5: { + internal_map6: {test1: 1, test2: 2}, + internal_map7: { + value: 1 + } + } +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("map"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 21); + UNIT_ASSERT_VALUES_EQUAL(end, 246); + } + + { + TString str = R"( +a: foo +map: !!map # comment +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("map"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 11); + UNIT_ASSERT_VALUES_EQUAL(end, 11); + } + + { + TString str = R"( +a: foo +map: {} # comment +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("map"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 13); + UNIT_ASSERT_VALUES_EQUAL(end, 15); + } + + { + TString str = R"( +a: foo +map: + value: 1 +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("map"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 15); + UNIT_ASSERT_VALUES_EQUAL(end, 23); + } + } + + Y_UNIT_TEST(SequenceMark) { + { + TString str = R"( +a: foo +seq: !!seq +- internal_map1: {} +- internal_seq2: + - internal_seq3: + - internal_seq4: + value: 1 + - internal_seq5: [ + internal_seq6: [{test1: 1}, {test2: 2}], + internal_seq7: [ + {value: 1} + ] + ] +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("seq"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 19); + UNIT_ASSERT_VALUES_EQUAL(end, 252); + } + + { + TString str = R"( +a: foo +seq: !!seq # comment +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("seq"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 11); + UNIT_ASSERT_VALUES_EQUAL(end, 11); + } + + { + TString str = R"( +a: foo +seq: [] # comment +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("seq"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 13); + UNIT_ASSERT_VALUES_EQUAL(end, 15); + } + + { + TString str = R"( +a: foo +seq: +- value: 1 +# comment +c: bar + )"; + + auto doc = NFyaml::TDocument::Parse(str); + + auto node = doc.Root().Map().at("seq"); + + auto begin = node.BeginMark().InputPos; + auto end = node.EndMark().InputPos; + + UNIT_ASSERT_VALUES_EQUAL(begin, 13); + UNIT_ASSERT_VALUES_EQUAL(end, 23); + } + } + } diff --git a/ydb/core/CMakeLists.txt b/ydb/core/CMakeLists.txt index 4ecd61df53..5493ccd3d7 100644 --- a/ydb/core/CMakeLists.txt +++ b/ydb/core/CMakeLists.txt @@ -16,6 +16,7 @@ add_subdirectory(client) add_subdirectory(cms) add_subdirectory(control) add_subdirectory(debug) +add_subdirectory(docapi) add_subdirectory(driver_lib) add_subdirectory(engine) add_subdirectory(erasure) diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index e740a2958a..26821f76dd 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -141,6 +141,7 @@ struct TAppData { NKikimrConfig::TCompactionConfig CompactionConfig; NKikimrConfig::TDomainsConfig DomainsConfig; NKikimrConfig::TBootstrap BootstrapConfig; + NKikimrConfig::TAwsCompatibilityConfig AwsCompatibilityConfig; std::optional<NKikimrSharedCache::TSharedCacheConfig> SharedCacheConfig; bool EnforceUserTokenRequirement = false; bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt index 6deaa41d0f..746cf2d3e2 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.darwin.txt @@ -49,6 +49,7 @@ target_sources(ydb-core-blobstorage-ut_blobstorage PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/main.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/sanitize_groups.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/scrub_fast.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/snapshots.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/space_check.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/sync.cpp diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt index 8c3720c83d..300e301488 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux-aarch64.txt @@ -52,6 +52,7 @@ target_sources(ydb-core-blobstorage-ut_blobstorage PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/main.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/sanitize_groups.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/scrub_fast.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/snapshots.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/space_check.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/sync.cpp diff --git a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt index edbb9626de..5de4c2c81e 100644 --- a/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/ut_blobstorage/CMakeLists.linux.txt @@ -54,6 +54,7 @@ target_sources(ydb-core-blobstorage-ut_blobstorage PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/main.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/sanitize_groups.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/scrub_fast.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/snapshots.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/space_check.cpp ${CMAKE_SOURCE_DIR}/ydb/core/blobstorage/ut_blobstorage/sync.cpp diff --git a/ydb/core/blobstorage/ut_blobstorage/scrub_fast.cpp b/ydb/core/blobstorage/ut_blobstorage/scrub_fast.cpp new file mode 100644 index 0000000000..b0e49b75fd --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/scrub_fast.cpp @@ -0,0 +1,79 @@ +#include <ydb/core/blobstorage/ut_blobstorage/lib/env.h> +#include <ydb/core/blobstorage/vdisk/scrub/scrub_actor.h> +#include <library/cpp/testing/unittest/registar.h> + +void Test() { + SetRandomSeed(1); + TEnvironmentSetup env{{ + .Erasure = TBlobStorageGroupType::Erasure4Plus2Block + }}; + auto& runtime = env.Runtime; + env.CreateBoxAndPool(); + env.SetScrubPeriodicity(TDuration::Seconds(60)); + env.Sim(TDuration::Minutes(1)); + auto groups = env.GetGroups(); + auto info = env.GetGroupInfo(groups[0]); + + TString data = TString::Uninitialized(8_MB); + memset(data.Detach(), 'X', data.size()); + TLogoBlobID id(1, 1, 1, 0, data.size(), 0); + + { // write data to group + TActorId sender = runtime->AllocateEdgeActor(1); + runtime->WrapInActorContext(sender, [&] { + SendToBSProxy(sender, info->GroupID, new TEvBlobStorage::TEvPut(id, data, TInstant::Max())); + }); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvPutResult>(sender); + UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK); + } + + auto checkReadable = [&](NKikimrProto::EReplyStatus status) { + TActorId sender = runtime->AllocateEdgeActor(1); + runtime->WrapInActorContext(sender, [&] { + SendToBSProxy(sender, info->GroupID, new TEvBlobStorage::TEvGet(id, 0, 0, TInstant::Max(), + NKikimrBlobStorage::EGetHandleClass::FastRead)); + }); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvGetResult>(sender); + UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK); + UNIT_ASSERT_VALUES_EQUAL(res->Get()->ResponseSz, 1); + auto& r = res->Get()->Responses[0]; + UNIT_ASSERT_VALUES_EQUAL(r.Status, status); + if (status == NKikimrProto::OK) { + UNIT_ASSERT_VALUES_EQUAL(r.Buffer, data); + } + }; + + checkReadable(NKikimrProto::OK); + + for (ui32 i = 0; i < info->GetTotalVDisksNum(); ++i) { + const TActorId vdiskActorId = info->GetActorId(i); + + ui32 nodeId, pdiskId; + std::tie(nodeId, pdiskId, std::ignore) = DecomposeVDiskServiceId(vdiskActorId); + auto it = env.PDiskMockStates.find(std::make_pair(nodeId, pdiskId)); + Y_VERIFY(it != env.PDiskMockStates.end()); + + const TActorId sender = runtime->AllocateEdgeActor(vdiskActorId.NodeId()); + env.Runtime->Send(new IEventHandle(vdiskActorId, sender, new TEvBlobStorage::TEvCaptureVDiskLayout), sender.NodeId()); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvCaptureVDiskLayoutResult>(sender); + + for (auto& item : res->Get()->Layout) { + using T = TEvBlobStorage::TEvCaptureVDiskLayoutResult; + if (item.Database == T::EDatabase::LogoBlobs && item.RecordType == T::ERecordType::HugeBlob) { + const TDiskPart& part = item.Location; + it->second->SetCorruptedArea(part.ChunkIdx, part.Offset, part.Offset + part.Size, true); + break; + } + } + + checkReadable(NKikimrProto::OK); + } + + env.Sim(TDuration::Seconds(60)); +} + +Y_UNIT_TEST_SUITE(ScrubFast) { + Y_UNIT_TEST(SingleBlob) { + Test(); + } +} diff --git a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_process.cpp b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_process.cpp index d78757e78a..43a2f3f863 100644 --- a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_process.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_process.cpp @@ -18,16 +18,16 @@ namespace NKikimr { } switch (TIngress::IngressMode(gtype)) { case TIngress::EMode::GENERIC: - if (i < gtype.TotalPartCount() && needed.Get(i)) { // only main replica - const TLogoBlobID partId(id, i + 1); - AddExtremeQuery(vdiskId, partId, deadline, gtype.PartSize(partId)); + ui32 maxSize; + maxSize = 0; + if (gtype.GetErasure() == TBlobStorageGroupType::ErasureMirror3dc) { + maxSize += gtype.PartSize(TLogoBlobID(id, i % 3 + 1)); } else { - ui32 maxSize = 0; - for (ui32 i = 0; i < gtype.TotalPartCount(); ++i) { - maxSize += gtype.PartSize(TLogoBlobID(id, i + 1)); + for (ui32 k = 0; k < gtype.TotalPartCount(); ++k) { + maxSize += i >= gtype.TotalPartCount() || k == i ? gtype.PartSize(TLogoBlobID(id, k + 1)) : 0; } - AddExtremeQuery(vdiskId, id, deadline, maxSize); } + AddExtremeQuery(vdiskId, id, deadline, maxSize); break; case TIngress::EMode::MIRROR3OF4: diff --git a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp index 6e4c2b6675..d07f17c064 100644 --- a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_queue.cpp @@ -38,9 +38,7 @@ namespace NKikimr { it->second.IsConnected = ev->Get()->IsConnected; STLOG(PRI_INFO, BS_VDISK_SCRUB, VDS29, VDISKP(LogPrefix, "BS_QUEUE state update"), (SelfId, SelfId()), (VDiskId, it->first), (IsConnected, it->second.IsConnected)); - if (it->second.IsConnected) { - EvaluateConnectionQuorum(); - } + EvaluateConnectionQuorum(); } void TBlobRecoveryActor::EvaluateConnectionQuorum() { diff --git a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp index ec2e24ff44..fcfbd6ed9f 100644 --- a/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/blob_recovery_request.cpp @@ -29,8 +29,7 @@ namespace NKikimr { // create timer to process deadlines if not yet created if (!WakeupScheduled) { - const TInstant now = TActivationContext::Now(); - Schedule(msg->Deadline - now, new TEvents::TEvWakeup); + Schedule(msg->Deadline, new TEvents::TEvWakeup); WakeupScheduled = true; } } @@ -57,7 +56,7 @@ namespace NKikimr { // reschedule timer if (deadline != TInstant::Max()) { - Schedule(deadline - now, new TEvents::TEvWakeup); + Schedule(deadline, new TEvents::TEvWakeup); } else { WakeupScheduled = false; } diff --git a/ydb/core/blobstorage/vdisk/scrub/restore_corrupted_blob_actor.cpp b/ydb/core/blobstorage/vdisk/scrub/restore_corrupted_blob_actor.cpp index c6e4a65833..af7e9b8282 100644 --- a/ydb/core/blobstorage/vdisk/scrub/restore_corrupted_blob_actor.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/restore_corrupted_blob_actor.cpp @@ -235,7 +235,7 @@ namespace NKikimr { } for (const auto& item : Items) { if (item.Status == NKikimrProto::UNKNOWN) { - return Schedule(TDuration::Seconds(1), new TEvIssueQuery); + return Schedule(TDuration::Seconds(5), new TEvIssueQuery); } } IssueQuery(); diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp b/ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp index ee81a7d832..f76005c8f5 100644 --- a/ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp @@ -35,7 +35,6 @@ namespace NKikimr { fFunc(TEvBlobStorage::EvRecoverBlob, ForwardToBlobRecoveryActor); hFunc(TEvRestoreCorruptedBlobResult, Handle); hFunc(TEvNonrestoredCorruptedBlobNotify, Handle); - cFunc(EvGenerateRestoreCorruptedBlobQuery, HandleGenerateRestoreCorruptedBlobQuery); hFunc(NPDisk::TEvLogResult, Handle); hFunc(NPDisk::TEvCutLog, Handle); @@ -73,9 +72,9 @@ namespace NKikimr { const TInstant end = start + TDuration::Seconds(30); do { Quantum(); - Send(ScrubCtx->SkeletonId, new TEvReportScrubStatus(UnreadableBlobs.size())); + Send(ScrubCtx->SkeletonId, new TEvReportScrubStatus(!UnreadableBlobs.empty())); } while (State && TActorCoroImpl::Now() < end); - Send(ScrubCtx->SkeletonId, new TEvReportScrubStatus(UnreadableBlobs.size())); + Send(ScrubCtx->SkeletonId, new TEvReportScrubStatus(!UnreadableBlobs.empty())); CommitStateUpdate(); } } catch (const TExDie&) { @@ -151,6 +150,9 @@ namespace NKikimr { } } + // validate currently held blobs + FilterUnreadableBlobs(*Snap, *GetBarriersEssence()); + ReleaseSnapshot(); if (const ui64 cookie = GenerateRestoreCorruptedBlobQuery()) { diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_impl.h b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_impl.h index 3097197cc2..bbef1aa010 100644 --- a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_impl.h +++ b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_impl.h @@ -51,10 +51,6 @@ namespace NKikimr { bool Success = false; - enum { - EvGenerateRestoreCorruptedBlobQuery = EventSpaceBegin(TEvents::ES_PRIVATE), - }; - private: struct TUnreadableBlobState { NMatrix::TVectorType UnreadableParts; // parts we're going to recover from peer disks @@ -78,6 +74,8 @@ namespace NKikimr { void AddUnreadableParts(const TLogoBlobID& fullId, NMatrix::TVectorType corrupted, TDiskPart corruptedPart); void UpdateUnreadableParts(const TLogoBlobID& fullId, NMatrix::TVectorType corrupted, TDiskPart corruptedPart); void UpdateReadableParts(const TLogoBlobID& fullId, NMatrix::TVectorType readable); + void Handle(TEvTakeHullSnapshotResult::TPtr ev); + void FilterUnreadableBlobs(THullDsSnap& snap, TBarriersSnapshot::TBarriersEssence& barriers); ui64 GenerateRestoreCorruptedBlobQuery(); void Handle(TAutoPtr<TEventHandle<TEvRestoreCorruptedBlobResult>> ev); diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_snapshot.cpp b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_snapshot.cpp index 14054d2bde..155421f117 100644 --- a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_snapshot.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_snapshot.cpp @@ -7,7 +7,7 @@ namespace NKikimr { CurrentState = TStringBuilder() << "waiting for Hull snapshot"; auto res = WaitForSpecificEvent<TEvTakeHullSnapshotResult>(); Snap.emplace(std::move(res->Get()->Snap)); - Snap->BarriersSnap.Destroy(); // barriers are not needed for operation + Snap->BlocksSnap.Destroy(); // blocks are not needed for operation } void TScrubCoroImpl::ReleaseSnapshot() { diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_unreadable.cpp b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_unreadable.cpp index 7d71701926..200796b53c 100644 --- a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_unreadable.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_unreadable.cpp @@ -135,8 +135,8 @@ namespace NKikimr { } } if (when != TInstant::Max()) { - GetActorSystem()->Schedule(when, new IEventHandle(EvGenerateRestoreCorruptedBlobQuery, 0, SelfActorId, - {}, nullptr, 0)); + GetActorSystem()->Schedule(when, new IEventHandle(ScrubCtx->SkeletonId, SelfActorId, + new TEvTakeHullSnapshot(false))); GenerateRestoreCorruptedBlobQueryScheduled = true; } } @@ -151,10 +151,37 @@ namespace NKikimr { GenerateRestoreCorruptedBlobQuery(); } - void TScrubCoroImpl::HandleGenerateRestoreCorruptedBlobQuery() { + void TScrubCoroImpl::Handle(TEvTakeHullSnapshotResult::TPtr ev) { Y_VERIFY(GenerateRestoreCorruptedBlobQueryScheduled); GenerateRestoreCorruptedBlobQueryScheduled = false; + + auto& snap = ev->Get()->Snap; + auto barriers = snap.BarriersSnap.CreateEssence(snap.HullCtx); + FilterUnreadableBlobs(snap, *barriers); + GenerateRestoreCorruptedBlobQuery(); } + void TScrubCoroImpl::FilterUnreadableBlobs(THullDsSnap& snap, TBarriersSnapshot::TBarriersEssence& barriers) { + TLevelIndexSnapshot::TForwardIterator iter(snap.HullCtx, &snap.LogoBlobsSnap); + TIndexRecordMerger merger(Info->Type); + + for (auto it = UnreadableBlobs.begin(); it != UnreadableBlobs.end(); ) { + const TLogoBlobID id = it->first; + ++it; + + bool keepData = false; + if (iter.Seek(id); iter.Valid() && iter.GetCurKey().LogoBlobID() == id) { + iter.PutToMerger(&merger); + merger.Finish(); + keepData = barriers.Keep(id, merger.GetMemRec(), merger.GetMemRecsMerged(), snap.HullCtx->AllowKeepFlags).KeepData; + merger.Clear(); + } + + if (!keepData) { + DropGarbageBlob(id); + } + } + } + } // NKikimr diff --git a/ydb/core/cms/CMakeLists.darwin.txt b/ydb/core/cms/CMakeLists.darwin.txt index ccf01d6c41..1df0fef60c 100644 --- a/ydb/core/cms/CMakeLists.darwin.txt +++ b/ydb/core/cms/CMakeLists.darwin.txt @@ -99,14 +99,15 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js + ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -136,6 +137,7 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js + ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -151,6 +153,7 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js + cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -180,6 +183,7 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js + cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/CMakeLists.linux-aarch64.txt b/ydb/core/cms/CMakeLists.linux-aarch64.txt index b9e51b0dc5..91275df5d1 100644 --- a/ydb/core/cms/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/CMakeLists.linux-aarch64.txt @@ -101,14 +101,15 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js + ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -138,6 +139,7 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js + ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -153,6 +155,7 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js + cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -182,6 +185,7 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js + cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/CMakeLists.linux.txt b/ydb/core/cms/CMakeLists.linux.txt index b9e51b0dc5..91275df5d1 100644 --- a/ydb/core/cms/CMakeLists.linux.txt +++ b/ydb/core/cms/CMakeLists.linux.txt @@ -101,14 +101,15 @@ target_link_libraries(ydb-core-cms.global PUBLIC tools-enum_parser-enum_serialization_runtime ) target_sources(ydb-core-cms.global PRIVATE - ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp ) resources(ydb-core-cms.global - ${CMAKE_BINARY_DIR}/ydb/core/cms/5aa7f9361b96d3de658aa5b60e0263fd.cpp + ${CMAKE_BINARY_DIR}/ydb/core/cms/f13dea7eff082ee71bdf3de5a8cf9130.cpp INPUTS ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/index.html ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms.js + ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/config_dispatcher.css ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/cms_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/console_log.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/common.css @@ -138,6 +139,7 @@ resources(ydb-core-cms.global ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/require.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/jquery.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/main.js + ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/configs_dispatcher_main.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/question-circle.svg ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/bootstrap.bundle.min.js ${CMAKE_SOURCE_DIR}/ydb/core/cms/ui/ext/theme.blue.css @@ -153,6 +155,7 @@ resources(ydb-core-cms.global cms/ui/index.html cms/ui/cms.css cms/ui/cms.js + cms/ui/config_dispatcher.css cms/ui/cms_log.js cms/ui/console_log.js cms/ui/common.css @@ -182,6 +185,7 @@ resources(ydb-core-cms.global cms/ui/ext/require.min.js cms/ui/ext/jquery.min.js cms/ui/main.js + cms/ui/configs_dispatcher_main.js cms/ui/ext/question-circle.svg cms/ui/ext/bootstrap.bundle.min.js cms/ui/ext/theme.blue.css diff --git a/ydb/core/cms/cluster_info.h b/ydb/core/cms/cluster_info.h index df5c3b5bf3..406f4d14b5 100644 --- a/ydb/core/cms/cluster_info.h +++ b/ydb/core/cms/cluster_info.h @@ -619,13 +619,12 @@ public: } void RollbackOperations() { - for (auto operation : Log) { - if (operation->Type == OPERATION_TYPE_ROLLBACK_POINT) { - Log.pop_back(); - break; - } + while (!Log.empty() && Log.back()->Type != OPERATION_TYPE_ROLLBACK_POINT) { + Log.back()->Undo(); + Log.pop_back(); + } - operation->Undo(); + if (!Log.empty() && Log.back()->Type == OPERATION_TYPE_ROLLBACK_POINT) { Log.pop_back(); } } diff --git a/ydb/core/cms/cms.cpp b/ydb/core/cms/cms.cpp index 54f5ff701c..0000852a8b 100644 --- a/ydb/core/cms/cms.cpp +++ b/ydb/core/cms/cms.cpp @@ -85,10 +85,7 @@ void TCms::ProcessInitQueue(const TActorContext &ctx) void TCms::SubscribeForConfig(const TActorContext &ctx) { - ctx.Register(NConsole::CreateConfigSubscriber(TabletID(), - {(ui32)NKikimrConsole::TConfigItem::CmsConfigItem}, - "", - ctx.SelfID)); + NConsole::SubscribeViaConfigDispatcher(ctx, {(ui32)NKikimrConsole::TConfigItem::CmsConfigItem}, ctx.SelfID); } void TCms::AdjustInfo(TClusterInfoPtr &info, const TActorContext &ctx) const @@ -980,6 +977,8 @@ void TCms::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::CMS, "TCms::Cleanup"); + NConsole::UnsubscribeViaConfigDispatcher(ctx, ctx.SelfID); + if (State->Sentinel) ctx.Send(State->Sentinel, new TEvents::TEvPoisonPill); } @@ -1979,7 +1978,14 @@ void TCms::Handle(TEvCms::TEvGetSentinelStateRequest::TPtr &ev, const TActorCont void TCms::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx) { - Execute(CreateTxUpdateConfig(ev), ctx); + if (ev->Get()->Record.HasLocal() && ev->Get()->Record.GetLocal()) { + Execute(CreateTxUpdateConfig(ev), ctx); + } else { + // ignore and immediately ack messages from old persistent console subscriptions + auto response = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(); + response->Record.MutableConfigId()->CopyFrom(ev->Get()->Record.GetConfigId()); + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } } void TCms::Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev, diff --git a/ydb/core/cms/cms_impl.h b/ydb/core/cms/cms_impl.h index cfe600bfcf..e332a08c8b 100644 --- a/ydb/core/cms/cms_impl.h +++ b/ydb/core/cms/cms_impl.h @@ -12,6 +12,7 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/base/statestorage_impl.h> #include <ydb/core/cms/console/console.h> +#include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/protos/counters_cms.pb.h> #include <ydb/core/tablet/tablet_counters_protobuf.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> @@ -204,6 +205,8 @@ private: TEvCms::TEvManageNotificationResponse>)); IgnoreFunc(TEvTabletPipe::TEvServerConnected); IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { @@ -252,6 +255,8 @@ private: HFunc(TEvTabletPipe::TEvClientConnected, Handle); IgnoreFunc(TEvTabletPipe::TEvServerConnected); IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { diff --git a/ydb/core/cms/cms_state.h b/ydb/core/cms/cms_state.h index 1c604f3445..9aaecf4f97 100644 --- a/ydb/core/cms/cms_state.h +++ b/ydb/core/cms/cms_state.h @@ -41,6 +41,9 @@ struct TCmsState : public TAtomicRefCount<TCmsState> { // CMS config. TCmsConfig Config; + // CMS config proto cache + NKikimrCms::TCmsConfig ConfigProto; + // Cluster info. It's not initialized on state creation. // Updated by event from info collector by rewritting // pointer. Therefore pointer shouldnt be preserved diff --git a/ydb/core/cms/cms_tx_load_state.cpp b/ydb/core/cms/cms_tx_load_state.cpp index bacfbdbaba..f6c0adaa5b 100644 --- a/ydb/core/cms/cms_tx_load_state.cpp +++ b/ydb/core/cms/cms_tx_load_state.cpp @@ -58,6 +58,7 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::CMS, "Using default config"); } + state->ConfigProto = config; state->Config.Deserialize(config); if (!logRowset.EndOfSet()) diff --git a/ydb/core/cms/cms_tx_update_config.cpp b/ydb/core/cms/cms_tx_update_config.cpp index 033a9cf28e..0b04136ffc 100644 --- a/ydb/core/cms/cms_tx_update_config.cpp +++ b/ydb/core/cms/cms_tx_update_config.cpp @@ -6,14 +6,13 @@ namespace NKikimr::NCms { class TCms::TTxUpdateConfig : public TTransactionBase<TCms> { public: - TTxUpdateConfig(TCms *self, - const NKikimrCms::TCmsConfig &config, - TAutoPtr<IEventHandle> response, - ui64 subscriptionId = 0) + TTxUpdateConfig( + TCms *self, + const NKikimrCms::TCmsConfig &config, + TAutoPtr<IEventHandle> response) : TBase(self) , Config(config) , Response(response) - , SubscriptionId(subscriptionId) , Modify(false) { } @@ -25,18 +24,13 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::CMS, "TTxUpdateConfig Execute"); - if (SubscriptionId != Self->ConfigSubscriptionId) { - LOG_ERROR_S(ctx, NKikimrServices::CMS, - "Config subscription id mismatch (" << SubscriptionId - << " vs expected " << Self->ConfigSubscriptionId << ")"); - return true; - } - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::Param>().Key(1) - .Update<Schema::Param::Config>(Config); + if (!google::protobuf::util::MessageDifferencer::Equals(Config, Self->State->ConfigProto)) { + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::Param>().Key(1) + .Update<Schema::Param::Config>(Config); - Modify = true; + Modify = true; + } return true; } @@ -46,14 +40,15 @@ public: LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxUpdateConfig Complete"); if (Modify) { + Self->State->ConfigProto = Config; Self->State->Config.Deserialize(Config); LOG_DEBUG_S(ctx, NKikimrServices::CMS, "Updated config: " << Config.ShortDebugString()); - - ctx.Send(Response.Release()); } + ctx.Send(Response.Release()); + if (Self->State->Config.SentinelConfig.Enable) { if (!Self->State->Sentinel) { Self->State->Sentinel = Self->RegisterWithSameMailbox(CreateSentinel(Self->State)); @@ -69,7 +64,6 @@ public: private: NKikimrCms::TCmsConfig Config; TAutoPtr<IEventHandle> Response; - ui64 SubscriptionId; bool Modify; }; @@ -78,13 +72,11 @@ ITransaction *TCms::CreateTxUpdateConfig(TEvConsole::TEvConfigNotificationReques auto &rec = ev->Get()->Record; auto response = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(); - response->Record.SetSubscriptionId(rec.GetSubscriptionId()); response->Record.MutableConfigId()->CopyFrom(rec.GetConfigId()); return new TTxUpdateConfig(this, rec.GetConfig().GetCmsConfig(), - new IEventHandle(ev->Sender, ev->Recipient, - response.Release(), 0, ev->Cookie), - rec.GetSubscriptionId()); + new IEventHandle(ev->Sender, ev->Recipient, response.Release(), 0, ev->Cookie) + ); } ITransaction *TCms::CreateTxUpdateConfig(TEvCms::TEvSetConfigRequest::TPtr &ev) @@ -93,9 +85,8 @@ ITransaction *TCms::CreateTxUpdateConfig(TEvCms::TEvSetConfigRequest::TPtr &ev) = new TEvCms::TEvSetConfigResponse; response->Record.MutableStatus()->SetCode(NKikimrCms::TStatus::OK); return new TTxUpdateConfig(this, ev->Get()->Record.GetConfig(), - new IEventHandle(ev->Sender, ev->Recipient, - response.Release(), 0, ev->Cookie), - ConfigSubscriptionId); + new IEventHandle(ev->Sender, ev->Recipient, response.Release(), 0, ev->Cookie) + ); } } // namespace NKikimr::NCms diff --git a/ydb/core/cms/cms_ut.cpp b/ydb/core/cms/cms_ut.cpp index 4d4b6e7960..6994d0bcd2 100644 --- a/ydb/core/cms/cms_ut.cpp +++ b/ydb/core/cms/cms_ut.cpp @@ -1476,6 +1476,37 @@ Y_UNIT_TEST_SUITE(TCmsTest) { } UNIT_ASSERT_VALUES_EQUAL(env.ProcessQueueCount, RequestsCount); } + + Y_UNIT_TEST(TestLogOperationsRollback) + { + TCmsTestEnv env(24); + + const ui32 requestsCount = 8; + + env.SetLimits(0, 0, 3, 0); + env.CreateDefaultCmsPipe(); + for (ui32 i = 0; i < requestsCount; ++i) { + auto req = MakePermissionRequest("user", false, true, false, + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(i), 60000000, "storage"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(i + 8), 60000000, "storage"), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(i + 16), 60000000, "storage")); + req->Record.SetDuration(60000000); + req->Record.SetAvailabilityMode(MODE_KEEP_AVAILABLE); + + env.SendToCms(req.Release()); + } + env.DestroyDefaultCmsPipe(); + + // Check responses order + for (ui32 i = 0; i < requestsCount; ++i) { + TAutoPtr<IEventHandle> handle; + auto reply = env.GrabEdgeEventRethrow<TEvCms::TEvPermissionResponse>(handle); + const auto &rec = reply->Record; + + UNIT_ASSERT_VALUES_EQUAL(rec.permissions_size(), 3); + UNIT_ASSERT_VALUES_EQUAL(rec.status().code(), TStatus::ALLOW); + } + } } } // NCmsTest } // NKikimr diff --git a/ydb/core/cms/console/CMakeLists.darwin.txt b/ydb/core/cms/console/CMakeLists.darwin.txt index 40a351c697..6bc2ce0311 100644 --- a/ydb/core/cms/console/CMakeLists.darwin.txt +++ b/ydb/core/cms/console/CMakeLists.darwin.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -37,7 +38,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp @@ -48,14 +48,17 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console_tenants_manager.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__add_config_subscription.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__alter_tenant.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__apply_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__replace_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__set_yaml_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__cleanup_subscriptions.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__configure.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__create_tenant.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__drop_yaml_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__init_scheme.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__load_state.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_log_tail.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_yaml_metadata.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__log_cleanup.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__remove_computational_units.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__remove_config_subscription.cpp diff --git a/ydb/core/cms/console/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/CMakeLists.linux-aarch64.txt index 94e500e8ec..ff0aa6a821 100644 --- a/ydb/core/cms/console/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/CMakeLists.linux-aarch64.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -38,7 +39,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp @@ -49,14 +49,17 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console_tenants_manager.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__add_config_subscription.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__alter_tenant.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__apply_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__replace_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__set_yaml_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__cleanup_subscriptions.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__configure.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__create_tenant.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__drop_yaml_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__init_scheme.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__load_state.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_log_tail.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_yaml_metadata.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__log_cleanup.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__remove_computational_units.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__remove_config_subscription.cpp diff --git a/ydb/core/cms/console/CMakeLists.linux.txt b/ydb/core/cms/console/CMakeLists.linux.txt index 94e500e8ec..ff0aa6a821 100644 --- a/ydb/core/cms/console/CMakeLists.linux.txt +++ b/ydb/core/cms/console/CMakeLists.linux.txt @@ -7,6 +7,7 @@ add_subdirectory(ut) +add_subdirectory(util) add_subdirectory(validators) add_subdirectory(yaml_config) @@ -38,7 +39,6 @@ target_link_libraries(core-cms-console PUBLIC ) target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_helpers.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/config_index.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_cache.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/configs_dispatcher.cpp @@ -49,14 +49,17 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console_tenants_manager.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__add_config_subscription.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__alter_tenant.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__apply_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__replace_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__set_yaml_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__cleanup_subscriptions.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__configure.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__create_tenant.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__drop_yaml_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__init_scheme.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__load_state.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_log_tail.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_yaml_config.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__get_yaml_metadata.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__log_cleanup.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__remove_computational_units.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/console__remove_config_subscription.cpp diff --git a/ydb/core/cms/console/config_helpers.cpp b/ydb/core/cms/console/config_helpers.cpp index 7b436f65bc..5c61225d86 100644 --- a/ydb/core/cms/console/config_helpers.cpp +++ b/ydb/core/cms/console/config_helpers.cpp @@ -1,4 +1,5 @@ #include "config_helpers.h" +#include "configs_dispatcher.h" #include "console.h" #include "util.h" @@ -446,4 +447,25 @@ IActor *CreateSubscriptionEraser(ui64 subscriptionId, return new TConfigHelper(subscriptionId, owner, cookie); } +void SubscribeViaConfigDispatcher(const TActorContext &ctx, + const TVector<ui32> &configItemKinds, + TActorId owner, + ui64 cookie) +{ + ctx.Send( + MakeConfigsDispatcherID(ctx.SelfID.NodeId()), + new TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(configItemKinds, owner), + cookie); +} + +void UnsubscribeViaConfigDispatcher(const TActorContext &ctx, + TActorId owner, + ui64 cookie) +{ + ctx.Send( + MakeConfigsDispatcherID(ctx.SelfID.NodeId()), + new TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest(owner), + cookie); +} + } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/config_helpers.h b/ydb/core/cms/console/config_helpers.h index 0ff594779a..e02a7bea79 100644 --- a/ydb/core/cms/console/config_helpers.h +++ b/ydb/core/cms/console/config_helpers.h @@ -71,6 +71,20 @@ IActor *CreateConfigSubscriber(TActorId serviceId, ui64 cookie = 0); /** + * These functions will make subscription through configs dispatcher + * Those subscriptions handle both yaml and non-yaml configs (not in same subscriprion) + * handle all deduplication, and reconnects + * internally new configs dispatcher uses InMemorySubscriprion's + */ +void SubscribeViaConfigDispatcher(const TActorContext &ctx, + const TVector<ui32> &configItemKinds, + TActorId owner, + ui64 cookie = 0); +void UnsubscribeViaConfigDispatcher(const TActorContext &ctx, + TActorId owner, + ui64 cookie = 0); + +/** * Subscription eraser is used to remove config subscriptions by ID. If owner is * specified then TEvConsole::TEvRemoveConfigSubscriptionRepsonse event is * forwared to it. diff --git a/ydb/core/cms/console/configs_cache.cpp b/ydb/core/cms/console/configs_cache.cpp index 49fedcba92..abe0ac9385 100644 --- a/ydb/core/cms/console/configs_cache.cpp +++ b/ydb/core/cms/console/configs_cache.cpp @@ -96,7 +96,7 @@ void TConfigsCache::Bootstrap(const TActorContext &ctx) { kinds.push_back(kind); } - auto client = CreateConfigsSubscriber(SelfId(), kinds, CurrentConfig); + auto client = CreateConfigsSubscriber(SelfId(), kinds, CurrentConfig, 1); SubscriptionClient = ctx.RegisterWithSameMailbox(client); Become(&TThis::StateWork); diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 9a125724d3..c0bdbee2ca 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -1,14 +1,22 @@ #include "config_helpers.h" -#include "config_index.h" #include "configs_dispatcher.h" +#include "console_configs_subscriber.h" #include "console.h" #include "http.h" +#include "util.h" +#include <ydb/core/cms/console/util/config_index.h> +#include <ydb/core/cms/console/yaml_config/util.h> +#include <ydb/core/cms/console/yaml_config/yaml_config.h> #include <ydb/core/mind/tenant_pool.h> #include <ydb/core/mon/mon.h> #include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/mon.h> +#include <library/cpp/actors/interconnect/interconnect.h> +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_writer.h> #include <util/generic/bitmap.h> #include <util/generic/ptr.h> @@ -27,17 +35,45 @@ namespace NKikimr::NConsole { -namespace { +const THashSet<ui32> DYNAMIC_KINDS({ + (ui32)NKikimrConsole::TConfigItem::ActorSystemConfigItem, + (ui32)NKikimrConsole::TConfigItem::BootstrapConfigItem, + (ui32)NKikimrConsole::TConfigItem::CmsConfigItem, + (ui32)NKikimrConsole::TConfigItem::CompactionConfigItem, + (ui32)NKikimrConsole::TConfigItem::ConfigsDispatcherConfigItem, + (ui32)NKikimrConsole::TConfigItem::FeatureFlagsItem, + (ui32)NKikimrConsole::TConfigItem::HiveConfigItem, + (ui32)NKikimrConsole::TConfigItem::ImmediateControlsConfigItem, + (ui32)NKikimrConsole::TConfigItem::LogConfigItem, + (ui32)NKikimrConsole::TConfigItem::MonitoringConfigItem, + (ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, + (ui32)NKikimrConsole::TConfigItem::NodeBrokerConfigItem, + (ui32)NKikimrConsole::TConfigItem::SchemeShardConfigItem, + (ui32)NKikimrConsole::TConfigItem::SharedCacheConfigItem, + (ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem, + (ui32)NKikimrConsole::TConfigItem::TableServiceConfigItem, + (ui32)NKikimrConsole::TConfigItem::TenantPoolConfigItem, + (ui32)NKikimrConsole::TConfigItem::TenantSlotBrokerConfigItem, +}); + +const THashSet<ui32> NON_YAML_KINDS({ + (ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, +}); class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> { private: using TBase = TActorBootstrapped<TConfigsDispatcher>; struct TConfig { - TConfigId ConfigId; + NKikimrConfig::TConfigVersion Version; NKikimrConfig::TAppConfig Config; }; + struct TYamlVersion { + ui64 Version; + TMap<ui64, ui64> VolatileVersions; + }; + /** * Structure to describe configs subscription shared by multiple * dispatcher subscribers. @@ -45,20 +81,31 @@ private: struct TSubscription : public TThrRefBase { using TPtr = TIntrusivePtr<TSubscription>; - // ID of corresponding subscription in CMS. Zero value means - // we haven't received subscription confirmation from CMS yet. - ui64 SubscriptionId = 0; TDynBitMap Kinds; - THashSet<TActorId> Subscribers; + THashMap<TActorId, ui64> Subscribers; + + // Set to true for all yaml kinds. + // Some 'legacy' kinds, which is usually managed by some automation e.g. NetClassifierDistributableConfigItem + // Left this field false and consume old console configs + bool Yaml = false; + // Set to true if there were no config update notifications - // processed for this subscription. - bool FirstUpdate = true; // Last config which was delivered to all subscribers. TConfig CurrentConfig; + + // If any yaml config delivered to all subscribers and acknowleged by them + // This field is set to version from this yaml config + std::optional<TYamlVersion> YamlVersion; + // Config update which is currently delivered to subscribers. - TEvConsole::TEvConfigNotificationRequest::TPtr UpdateInProcess; + THolder<TEvConsole::TEvConfigNotificationRequest> UpdateInProcess = nullptr; + NKikimrConfig::TConfigVersion UpdateInProcessConfigVersion; + ui64 UpdateInProcessCookie; + std::optional<TYamlVersion> UpdateInProcessYamlVersion; + // Subscribers who didn't respond yet to the latest config update. THashSet<TActorId> SubscribersToUpdate; + }; /** @@ -72,7 +119,7 @@ private: TActorId Subscriber; THashSet<TSubscription::TPtr> Subscriptions; - TConfigId CurrentConfigId; + NKikimrConfig::TConfigVersion CurrentConfigVersion; }; public: @@ -81,139 +128,86 @@ public: return NKikimrServices::TActivity::CONFIGS_DISPATCHER_ACTOR; } - TConfigsDispatcher(const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels); + TConfigsDispatcher( + const NKikimrConfig::TAppConfig &config, + const TMap<TString, TString> &labels, + const NKikimrConfig::TAppConfig &initialCmsConfig, + const NKikimrConfig::TAppConfig &initialCmsYamlConfig); void Bootstrap(); void EnqueueEvent(TAutoPtr<IEventHandle> &ev); void ProcessEnqueuedEvents(); - TDynBitMap KindsToBitMap(const TVector<ui32> &kinds) const; - TString KindsToString(const TDynBitMap &kinds) const; - TVector<ui32> KindsToVector(const TDynBitMap &kinds) const; - - /** - * Overwrite specified protobuf fields with values from - * another protobuf. It's assumed that source config doesn't - * have fields not listed in kinds. - */ - void ReplaceConfigItems(const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds) const; - bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs); + void SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber); - TSubscription::TPtr FindSubscription(ui64 id); + TSubscription::TPtr FindSubscription(const TActorId &subscriber); TSubscription::TPtr FindSubscription(const TDynBitMap &kinds); TSubscriber::TPtr FindSubscriber(TActorId aid); - void SendNotificationResponse(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); - - void MaybeSendNotificationResponse(TSubscription::TPtr subscription); - - void CreateSubscriberActor(ui32 kind, bool replace); - void CreateSubscriberActor(const TDynBitMap &kinds, bool replace); - /** - * Send config notification to a subscriber. Called for subscriptions - * having config update being processed. - */ - - void SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber); - /** - * Remove subscriber and all his subscriptions. - */ - void RemoveSubscriber(TSubscriber::TPtr subscriber); - - /** - * Remove subscription from own data and CMS. It should be called - * for confirmed CMS subscriptions which have no more local - * subscribers. - */ - void RemoveSubscription(TSubscription::TPtr subscription); - - /** - * Create subscription for subscriber. If subscription with similar - * config kinds already exists then just re-use it. Otherwise - * create a new one. If existing subscription has some config received - * then deliver it to the new subscriber. - */ - void AddSubscription(TActorId subscriber, const TDynBitMap &kinds, bool replace); - - /** - * This is called on start and on tenant change to clean up old config - * subscriptions. It also adds subscription for own config. - */ - void CleanUpSubscriptions(); - /** - * Process successfull subscription registration in CMS. Send - * corresponsing notifications to subscribers. If no more subscribers - * left for this subscription then remove it. - */ - - void ProcessAddedSubscription(TSubscription::TPtr subscription, ui64 id); - - /** - * This method is used to process notifications sent to self. - */ - void ProcessLocalCacheUpdate(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); + void UpdateYamlVersion(const TSubscription::TPtr &kinds) const; + NKikimrConfig::TAppConfig ParseYamlProtoConfig(); + void Handle(NMon::TEvHttpInfo::TPtr &ev); + void Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev); + void Handle(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev); + void Handle(TEvConsole::TEvConfigSubscriptionError::TPtr &ev); + void Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); void Handle(TEvConfigsDispatcher::TEvGetConfigRequest::TPtr &ev); void Handle(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest::TPtr &ev); - void Handle(TEvConsole::TEvAddConfigSubscriptionResponse::TPtr &ev); - void Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); + void Handle(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest::TPtr &ev); void Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev); - void Handle(TEvConsole::TEvGetNodeConfigResponse::TPtr &ev); - void Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev); - void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev); + void Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev); - /** - * Initial state when we just get status from tenant pool to collect assigned - * tenants. It's possible we start before tenant pool and therefore might have - * to retry request. - */ - STATEFN(StateInit) { - TRACE_EVENT(NKikimrServices::CONFIGS_DISPATCHER); - switch (ev->GetTypeRewrite()) { - hFuncTraced(NMon::TEvHttpInfo, Handle); - hFuncTraced(TEvTenantPool::TEvTenantPoolStatus, Handle); + void ReplyMonJson(TActorId mailbox); - default: - EnqueueEvent(ev); - break; - } - } - - /** - * In this state we remove all old service subscriptions and install a new - * one for own config. - */ - STATEFN(StateConfigure) { + STATEFN(StateInit) + { TRACE_EVENT(NKikimrServices::CONFIGS_DISPATCHER); switch (ev->GetTypeRewrite()) { + // Monitoring page hFuncTraced(NMon::TEvHttpInfo, Handle); - hFuncTraced(TEvConsole::TEvReplaceConfigSubscriptionsResponse, Handle); - + hFuncTraced(TEvInterconnect::TEvNodesInfo, Handle); + // Updates from console + hFuncTraced(TEvConsole::TEvConfigSubscriptionNotification, Handle); + hFuncTraced(TEvConsole::TEvConfigSubscriptionError, Handle); + // Events from clients + hFuncTraced(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest, Handle); + hFuncTraced(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest, Handle); + // Resolve + hFunc(TEvConsole::TEvGetNodeLabelsRequest, Handle); default: EnqueueEvent(ev); break; } } - /** - * Primary state for subscriptions and notifications processing. - */ - STATEFN(StateWork) { + STATEFN(StateWork) + { TRACE_EVENT(NKikimrServices::CONFIGS_DISPATCHER); switch (ev->GetTypeRewrite()) { + // Monitoring page hFuncTraced(NMon::TEvHttpInfo, Handle); + hFuncTraced(TEvInterconnect::TEvNodesInfo, Handle); + // Updates from console + hFuncTraced(TEvConsole::TEvConfigSubscriptionNotification, Handle); + hFuncTraced(TEvConsole::TEvConfigSubscriptionError, Handle); + // Events from clients hFuncTraced(TEvConfigsDispatcher::TEvGetConfigRequest, Handle); hFuncTraced(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest, Handle); - hFuncTraced(TEvConsole::TEvAddConfigSubscriptionResponse, Handle); + hFuncTraced(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest, Handle); hFuncTraced(TEvConsole::TEvConfigNotificationResponse, Handle); - hFuncTraced(TEvConsole::TEvConfigNotificationRequest, Handle); - hFuncTraced(TEvConsole::TEvGetNodeConfigResponse, Handle); - hFuncTraced(TEvTenantPool::TEvTenantPoolStatus, Handle); IgnoreFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + // Resolve + hFunc(TEvConsole::TEvGetNodeLabelsRequest, Handle); + // Ignore these console requests until we get rid of persistent subscriptions-related code + IgnoreFunc(TEvConsole::TEvAddConfigSubscriptionResponse); + IgnoreFunc(TEvConsole::TEvGetNodeConfigResponse); + // Pretend we got this + hFuncTraced(TEvConsole::TEvConfigNotificationRequest, Handle); default: Y_FAIL("unexpected event type: %" PRIx32 " event: %s", ev->GetTypeRewrite(), ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?"); @@ -221,34 +215,43 @@ public: } } + private: TMap<TString, TString> Labels; - TDeque<TAutoPtr<IEventHandle>> EventsQueue; - NKikimrConfig::TAppConfig InitialConfig; + const NKikimrConfig::TAppConfig InitialConfig; NKikimrConfig::TAppConfig CurrentConfig; - THashSet<TSubscription::TPtr> Subscriptions; - THashMap<ui64, TSubscription::TPtr> SubscriptionsById; + const NKikimrConfig::TAppConfig InitialCmsConfig; + const NKikimrConfig::TAppConfig InitialCmsYamlConfig; + ui64 NextRequestCookie; + TVector<TActorId> HttpRequests; + TActorId CommonSubscriptionClient; + TDeque<TAutoPtr<IEventHandle>> EventsQueue; + + THashMap<TActorId, TSubscription::TPtr> SubscriptionsBySubscriber; THashMap<TDynBitMap, TSubscription::TPtr> SubscriptionsByKinds; THashMap<TActorId, TSubscriber::TPtr> Subscribers; - // Messages that had an unknown subscription id at the time they are received - THashMap<ui64, TEvConsole::TEvConfigNotificationRequest::TPtr> OutOfOrderConfigNotifications; + TString YamlConfig; + TMap<ui64, TString> VolatileYamlConfigs; + TMap<ui64, size_t> VolatileYamlConfigHashes; + TString ResolvedYamlConfig; + TString ResolvedJsonConfig; + NKikimrConfig::TAppConfig YamlProtoConfig; + bool YamlConfigEnabled = false; - // Cookies are used to tie CMS requests to kinds they were generated for. - THashMap<ui64, TDynBitMap> RequestCookies; - ui64 NextRequestCookie; - THashSet<TString> CurrentTenants; - - // Structures to process config requests. - THashMap<ui64, THolder<IEventHandle>> ConfigRequests; - THashMap<TDynBitMap, std::shared_ptr<NKikimrConfig::TAppConfig>> ConfigsCache; }; -TConfigsDispatcher::TConfigsDispatcher(const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels) - : Labels(labels) - , InitialConfig(config) - , CurrentConfig(config) - , NextRequestCookie(Now().GetValue()) +TConfigsDispatcher::TConfigsDispatcher( + const NKikimrConfig::TAppConfig &config, + const TMap<TString, TString> &labels, + const NKikimrConfig::TAppConfig &initialCmsConfig, + const NKikimrConfig::TAppConfig &initialCmsYamlConfig) + : Labels(labels) + , InitialConfig(config) + , CurrentConfig(config) + , InitialCmsConfig(initialCmsConfig) + , InitialCmsYamlConfig(initialCmsYamlConfig) + , NextRequestCookie(Now().GetValue()) { } @@ -262,8 +265,16 @@ void TConfigsDispatcher::Bootstrap() mon->RegisterActorPage(actorsMonPage, "configs_dispatcher", "Configs Dispatcher", false, TlsActivationContext->ExecutorThread.ActorSystem, SelfId()); } + auto commonClient = CreateConfigsSubscriber( + SelfId(), + TVector<ui32>(DYNAMIC_KINDS.begin(), DYNAMIC_KINDS.end()), + CurrentConfig, + 0, + true, + 1); + CommonSubscriptionClient = RegisterWithSameMailbox(commonClient); + Become(&TThis::StateInit); - Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe); } void TConfigsDispatcher::EnqueueEvent(TAutoPtr<IEventHandle> &ev) @@ -282,673 +293,649 @@ void TConfigsDispatcher::ProcessEnqueuedEvents() } } -TDynBitMap TConfigsDispatcher::KindsToBitMap(const TVector<ui32> &kinds) const -{ - TDynBitMap result; - for (auto &kind : kinds) - result.Set(kind); - return result; -} - -TString TConfigsDispatcher::KindsToString(const TDynBitMap &kinds) const +void TConfigsDispatcher::SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber) { - TStringStream ss; - bool first = true; - Y_FOR_EACH_BIT(kind, kinds) { - ss << (first ? "" : ", ") << static_cast<NKikimrConsole::TConfigItem::EKind>(kind); - first = false; - } - return ss.Str(); -} + Y_VERIFY(subscription->UpdateInProcess); -TVector<ui32> TConfigsDispatcher::KindsToVector(const TDynBitMap &kinds) const -{ - TVector<ui32> res; - Y_FOR_EACH_BIT(kind, kinds) { - res.push_back(kind); - } - return res; -} + subscription->SubscribersToUpdate.insert(subscriber); -void TConfigsDispatcher::ReplaceConfigItems(const NKikimrConfig::TAppConfig &from, - NKikimrConfig::TAppConfig &to, - const TDynBitMap &kinds) const -{ - auto *desc = to.GetDescriptor(); - auto *reflection = to.GetReflection(); + auto notification = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); + notification->Record.CopyFrom(subscription->UpdateInProcess->Record); - Y_FOR_EACH_BIT(kind, kinds) { - auto *field = desc->FindFieldByNumber(kind); - if (field && reflection->HasField(to, field)) - reflection->ClearField(&to, field); - } + BLOG_TRACE("Send TEvConsole::TEvConfigNotificationRequest to " << subscriber + << ": " << notification->Record.ShortDebugString()); - to.MergeFrom(from); + Send(subscriber, notification.Release(), 0, subscription->UpdateInProcessCookie); } -bool TConfigsDispatcher::CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs) +TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(const TDynBitMap &kinds) { - TString str1, str2; - Y_PROTOBUF_SUPPRESS_NODISCARD lhs.SerializeToString(&str1); - Y_PROTOBUF_SUPPRESS_NODISCARD rhs.SerializeToString(&str2); - return (str1 == str2); -} + if (auto it = SubscriptionsByKinds.find(kinds); it != SubscriptionsByKinds.end()) + return it->second; -TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(ui64 id) -{ - auto it = SubscriptionsById.find(id); - if (it == SubscriptionsById.end()) - return nullptr; - return it->second; + return nullptr; } -TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(const TDynBitMap &kinds) +TConfigsDispatcher::TSubscription::TPtr TConfigsDispatcher::FindSubscription(const TActorId &id) { - auto it = SubscriptionsByKinds.find(kinds); - if (it == SubscriptionsByKinds.end()) - return nullptr; - return it->second; + if (auto it = SubscriptionsBySubscriber.find(id); it != SubscriptionsBySubscriber.end()) + return it->second; + + return nullptr; } TConfigsDispatcher::TSubscriber::TPtr TConfigsDispatcher::FindSubscriber(TActorId aid) { - auto it = Subscribers.find(aid); - if (it == Subscribers.end()) - return nullptr; - return it->second; + if (auto it = Subscribers.find(aid); it != Subscribers.end()) + return it->second; + + return nullptr; } -void TConfigsDispatcher::SendNotificationResponse(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) +NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig() { - const auto &rec = ev->Get()->Record; - auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); + NKikimrConfig::TAppConfig newYamlProtoConfig = {}; - BLOG_TRACE("Send TEvConfigNotificationResponse: " << resp->Record.ShortDebugString()); + try { + NYamlConfig::ResolveAndParseYamlConfig( + YamlConfig, + VolatileYamlConfigs, + Labels, + newYamlProtoConfig, + &ResolvedYamlConfig, + &ResolvedJsonConfig); + } catch (const yexception& ex) { + BLOG_ERROR("Got invalid config from console error# " << ex.what()); + } - Send(ev->Sender, resp.Release(), 0, ev->Cookie); + return newYamlProtoConfig; } -void TConfigsDispatcher::MaybeSendNotificationResponse(TSubscription::TPtr subscription) +void TConfigsDispatcher::Handle(NMon::TEvHttpInfo::TPtr &ev) { - if (!subscription->UpdateInProcess || !subscription->SubscribersToUpdate.empty()) + if (auto it = ev->Get()->Request.GetHeaders().FindHeader("Content-Type"); it && it->Value() == "application/json") { + ReplyMonJson(ev->Sender); return; + } - auto &rec = subscription->UpdateInProcess->Get()->Record; - subscription->CurrentConfig.Config.Swap(rec.MutableConfig()); - subscription->CurrentConfig.ConfigId.Load(rec.GetConfigId()); + if (HttpRequests.empty()) + Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes); - ReplaceConfigItems(subscription->CurrentConfig.Config, CurrentConfig, subscription->Kinds); + HttpRequests.push_back(ev->Sender); +} - BLOG_D("Got all confirmations for config update" - << " subscriptionid=" << subscription->SubscriptionId - << " configid=" << TConfigId(rec.GetConfigId()).ToString()); +void TConfigsDispatcher::ReplyMonJson(TActorId mailbox) { + TStringStream str; + str << NMonitoring::HTTPOKJSON; - SendNotificationResponse(subscription->UpdateInProcess); + NJson::TJsonValue response; + response.SetType(NJson::EJsonValueType::JSON_MAP); - subscription->UpdateInProcess = nullptr; -} + auto& labels = response["labels"]; + labels.SetType(NJson::EJsonValueType::JSON_ARRAY); + for (auto &[key, value] : Labels) { + NJson::TJsonValue label; + label.SetType(NJson::EJsonValueType::JSON_MAP); + label.InsertValue("name", key); + label.InsertValue("value", value); + labels.AppendValue(std::move(label)); + } -void TConfigsDispatcher::CreateSubscriberActor(ui32 kind, bool replace) -{ - TDynBitMap kinds; - kinds.Set(kind); - CreateSubscriberActor(kinds, replace); -} + response.InsertValue("yaml_config", YamlConfig); + response.InsertValue("resolved_json_config", NJson::ReadJsonFastTree(ResolvedJsonConfig, true)); + response.InsertValue("current_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(CurrentConfig, NYamlConfig::GetProto2JsonConfig()), true)); + response.InsertValue("initial_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(InitialConfig, NYamlConfig::GetProto2JsonConfig()), true)); + response.InsertValue("initial_cms_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(InitialCmsConfig, NYamlConfig::GetProto2JsonConfig()), true)); + response.InsertValue("initial_cms_yaml_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(InitialCmsYamlConfig, NYamlConfig::GetProto2JsonConfig()), true)); -void TConfigsDispatcher::CreateSubscriberActor(const TDynBitMap &kinds, bool replace) -{ - BLOG_D("Create new subscriber kinds=" << KindsToString(kinds)); - - auto *subscriber = CreateConfigSubscriber(MakeConfigsDispatcherID(SelfId().NodeId()), - KindsToVector(kinds), - SelfId(), - replace, - NextRequestCookie); - Register(subscriber); - RequestCookies[NextRequestCookie] = kinds; - ++NextRequestCookie; + NJson::WriteJson(&str, &response, {}); + + Send(mailbox, new NMon::TEvHttpInfoRes(str.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); } -void TConfigsDispatcher::SendUpdateToSubscriber(TSubscription::TPtr subscription, TActorId subscriber) +void TConfigsDispatcher::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) { - Y_VERIFY(subscription->SubscriptionId); - Y_VERIFY(subscription->UpdateInProcess); - - subscription->SubscribersToUpdate.insert(subscriber); - - auto notification = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); - notification->Record.CopyFrom(subscription->UpdateInProcess->Get()->Record); + const auto &rec = ev->Get()->Record; + auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); - BLOG_TRACE("Send TEvConsole::TEvConfigNotificationRequest to " << subscriber - << ": " << notification->Record.ShortDebugString()); + BLOG_TRACE("Send TEvConfigNotificationResponse: " << resp->Record.ShortDebugString()); - Send(subscriber, notification.Release(), 0, subscription->UpdateInProcess->Cookie); + Send(ev->Sender, resp.Release(), 0, ev->Cookie); } -void TConfigsDispatcher::RemoveSubscriber(TSubscriber::TPtr subscriber) +void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) { - BLOG_D("Remove subscriber " << subscriber->Subscriber); + Y_UNUSED(ev); + TStringStream str; + str << NMonitoring::HTTPOKHTML; + HTML(str) { + HEAD() { + str << "<link rel='stylesheet' href='../cms/ext/bootstrap.min.css'>" << Endl + << "<script language='javascript' type='text/javascript' src='../cms/ext/jquery.min.js'></script>" << Endl + << "<script language='javascript' type='text/javascript' src='../cms/ext/bootstrap.bundle.min.js'></script>" << Endl + << "<script language='javascript' type='text/javascript'>" << Endl + << "var nodeNames = ["; + + for (auto &node: ev->Get()->Nodes) { + str << "{'nodeName':'" << node.Host << "'}, "; + } - for (auto subscription : subscriber->Subscriptions) { - Y_VERIFY(subscription->Subscribers.contains(subscriber->Subscriber)); - subscription->Subscribers.erase(subscriber->Subscriber); + str << "];" << Endl + << "</script>" << Endl + << "<script src='../cms/ext/fuse.min.js'></script>" << Endl + << "<script src='../cms/common.js'></script>" << Endl + << "<script src='../cms/ext/fuzzycomplete.min.js'></script>" << Endl + << "<link rel='stylesheet' href='../cms/ext/fuzzycomplete.min.css'>" << Endl + << "<link rel='stylesheet' href='../cms/cms.css'>" << Endl + << "<script data-main='../cms/configs_dispatcher_main' src='../cms/ext/require.min.js'></script>" << Endl; - if (subscription->UpdateInProcess) { - subscription->SubscribersToUpdate.erase(subscriber->Subscriber); - MaybeSendNotificationResponse(subscription); } + NHttp::OutputStyles(str); + + DIV() { + OL_CLASS("breadcrumb") { + LI_CLASS("breadcrumb-item") { + str << "<a href='..' id='host-ref'>YDB Developer UI</a>" << Endl; + } + LI_CLASS("breadcrumb-item") { + str << "<a href='.'>Actors</a>" << Endl; + } + LI_CLASS("breadcrumb-item active") { + str << "Configs Dispatcher" << Endl; + } + } + } + + DIV_CLASS("container") { + DIV_CLASS("navbar navbar-expand-lg navbar-light bg-light") { + DIV_CLASS("navbar-collapse") { + UL_CLASS("navbar-nav mr-auto") { + LI_CLASS("nav-item") { + str << "<a class='nav-link' href=\"../cms?#page=yaml-config\">Console</a>" << Endl; + } + } + FORM_CLASS("form-inline my-2 my-lg-0") { + str << "<input type='text' id='nodePicker' class='form-control mr-sm-2' name='nodes' placeholder='Nodes...'>" << Endl; + str << "<a type='button' class='btn btn-primary my-2 my-sm-0' id='nodesGo'>Go</a>" << Endl; + } + } + } + DIV_CLASS("tab-left") { + COLLAPSED_REF_CONTENT("node-labels", "Node labels") { + PRE() { + for (auto &[key, value] : Labels) { + str << key << " = " << value << Endl; + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("state", "State") { + PRE() { + str << "SelfId: " << SelfId() << Endl; + auto s = CurrentStateFunc(); + str << "State: " << ( s == &TThis::StateWork ? "StateWork" + : s == &TThis::StateInit ? "StateInit" + : "Unknown" ) << Endl; + str << "YamlConfigEnabled: " << YamlConfigEnabled << Endl; + str << "Subscriptions: " << Endl; + for (auto &[kinds, subscription] : SubscriptionsByKinds) { + str << "- Kinds: " << KindsToString(kinds) << Endl + << " Subscription: " << Endl + << " Yaml: " << subscription->Yaml << Endl + << " Subscribers: " << Endl; + for (auto &[id, updates] : subscription->Subscribers) { + str << " - Actor: " << id << Endl; + str << " UpdatesSent: " << updates << Endl; + } + if (subscription->YamlVersion) { + str << " YamlVersion: " << subscription->YamlVersion->Version << ".["; + bool first = true; + for (auto &[id, hash] : subscription->YamlVersion->VolatileVersions) { + str << (first ? "" : ",") << id << "." << hash; + first = false; + } + str << "]" << Endl; + } else { + str << " CurrentConfigId: " << subscription->CurrentConfig.Version.ShortDebugString() << Endl; + } + str << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; + if (subscription->UpdateInProcess) { + str << " UpdateInProcess: " << subscription->UpdateInProcess->Record.ShortDebugString() << Endl + << " SubscribersToUpdate:"; + for (auto &id : subscription->SubscribersToUpdate) { + str << " " << id; + } + str << Endl; + str << " UpdateInProcessConfigVersion: " << subscription->UpdateInProcessConfigVersion.ShortDebugString() << Endl + << " UpdateInProcessCookie: " << subscription->UpdateInProcessCookie << Endl; + if (subscription->UpdateInProcessYamlVersion) { + str << " UpdateInProcessYamlVersion: " << subscription->UpdateInProcessYamlVersion->Version << Endl; + } + } + } + str << "Subscribers:" << Endl; + for (auto &[subscriber, _] : SubscriptionsBySubscriber) { + str << "- " << subscriber << Endl; + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("yaml-config", "YAML config") { + DIV() { + TAG(TH5) { + str << "Persistent Config" << Endl; + } + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-yaml-config"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-yaml-config"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-yaml-config"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"id", "yaml-config-item"}, {"name", "yaml-config-itemm"}}) { + str << YamlConfig; + } + } + str << "<hr/>" << Endl; + for (auto &[id, config] : VolatileYamlConfigs) { + DIV() { + TAG(TH5) { + str << "Volatile Config Id: " << id << Endl; + } + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + DIV_CLASS("yaml-config-item") { + str << config; + } + } + } + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-resolved-yaml-config"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-resolved-yaml-config"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"id", "resolved-yaml-config-item"}, {"name", "resolved-yaml-config-itemm"}}) { + str << ResolvedYamlConfig; + } + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("resolved-json-config", "Resolved JSON config") { + PRE() { + str << ResolvedJsonConfig << Endl; + } + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("yaml-proto-config", "YAML proto config") { + NHttp::OutputConfigHTML(str, YamlProtoConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("current-config", "Current config") { + NHttp::OutputConfigHTML(str, CurrentConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-config", "Initial config") { + NHttp::OutputConfigHTML(str, InitialConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") { + NHttp::OutputConfigHTML(str, InitialCmsConfig); + } + str << "<br />" << Endl; + COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") { + NHttp::OutputConfigHTML(str, InitialCmsYamlConfig); + } + } + } + } - // If there are no more subscribers using this subscription then - // it can be removed. Don't remove subscriptions which are not - // yet confirmed by CMS. - if (subscription->Subscribers.empty() && subscription->SubscriptionId) - RemoveSubscription(subscription); + for (auto &actor : HttpRequests) { + Send(actor, new NMon::TEvHttpInfoRes(str.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); } - Subscribers.erase(subscriber->Subscriber); + HttpRequests.clear(); } -void TConfigsDispatcher::RemoveSubscription(TSubscription::TPtr subscription) +void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev) { - Subscriptions.erase(subscription); - SubscriptionsById.erase(subscription->SubscriptionId); - SubscriptionsByKinds.erase(subscription->Kinds); + auto &rec = ev->Get()->Record; - BLOG_D("Remove subscription id=" << subscription->SubscriptionId - << " kinds=" << KindsToString(subscription->Kinds)); + CurrentConfig = rec.GetConfig(); - Register(CreateSubscriptionEraser(subscription->SubscriptionId)); -} + const auto& newYamlConfig = rec.GetYamlConfig(); -void TConfigsDispatcher::AddSubscription(TActorId subscriber, - const TDynBitMap &kinds, - bool replace) -{ - BLOG_D("Add subscription for " << subscriber << " kinds=" << KindsToString(kinds)); - - // If there is a subscription for required config kinds then - // re-use it for new subscriber. Otherwise create a new one. - auto subscription = FindSubscription(kinds); - if (!subscription) { - subscription = new TSubscription; - subscription->Kinds = kinds; + bool isYamlChanged = newYamlConfig != YamlConfig; - Subscriptions.insert(subscription); - SubscriptionsByKinds.emplace(kinds, subscription); + if (rec.VolatileConfigsSize() != VolatileYamlConfigs.size()) { + isYamlChanged = true; + } - CreateSubscriberActor(kinds, replace); + for (auto &volatileConfig : rec.GetVolatileConfigs()) { + if (auto it = VolatileYamlConfigHashes.find(volatileConfig.GetId()); + it == VolatileYamlConfigHashes.end() || it->second != THash<TString>()(volatileConfig.GetConfig())) { + isYamlChanged = true; + } } - subscription->Subscribers.insert(subscriber); - auto s = FindSubscriber(subscriber); - if (!s) { - s = new TSubscriber; - s->Subscriber = subscriber; - Subscribers.emplace(subscriber, s); + if (isYamlChanged) { + YamlConfig = newYamlConfig; + VolatileYamlConfigs.clear(); + VolatileYamlConfigHashes.clear(); + for (auto &volatileConfig : rec.GetVolatileConfigs()) { + VolatileYamlConfigs[volatileConfig.GetId()] = volatileConfig.GetConfig(); + VolatileYamlConfigHashes[volatileConfig.GetId()] = THash<TString>()(volatileConfig.GetConfig()); + } } - s->Subscriptions.insert(subscription); - // Non-zero subscription ID means there is an active CMS - // subscription and therefore we can respond to the subscriber - // immediately. Otherwise we should wait until CMS - // subscription request is complete. - if (subscription->SubscriptionId) { - Y_VERIFY(!replace); - auto resp = MakeHolder<TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse>(); + NKikimrConfig::TAppConfig newYamlProtoConfig = {}; - BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " - << subscriber); + bool yamlConfigTurnedOff = false; - Send(subscriber, resp.Release()); + if (!YamlConfig.empty() && isYamlChanged) { + newYamlProtoConfig = ParseYamlProtoConfig(); + bool wasYamlConfigEnabled = YamlConfigEnabled; + YamlConfigEnabled = newYamlProtoConfig.HasYamlConfigEnabled() && newYamlProtoConfig.GetYamlConfigEnabled(); + yamlConfigTurnedOff = wasYamlConfigEnabled && !YamlConfigEnabled; + } else if (YamlConfig.empty()) { + bool wasYamlConfigEnabled = YamlConfigEnabled; + YamlConfigEnabled = false; + yamlConfigTurnedOff = wasYamlConfigEnabled && !YamlConfigEnabled; + } else { + newYamlProtoConfig = YamlProtoConfig; } - // If there is an ongoing config update then include new subscriber into - // the process. - if (subscription->UpdateInProcess) { - Y_VERIFY(!replace); - SendUpdateToSubscriber(subscription, subscriber); - } else if (!subscription->FirstUpdate) { - // If subscription already had an update notification then send corresponding - // notification to the subscriber using current config. - Y_VERIFY(!replace); - Y_VERIFY(subscription->SubscriptionId); - auto notification = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); - notification->Record.SetSubscriptionId(subscription->SubscriptionId); - subscription->CurrentConfig.ConfigId.Serialize(*notification->Record.MutableConfigId()); - notification->Record.MutableConfig()->CopyFrom(subscription->CurrentConfig.Config); - - BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to "<< subscriber); + std::swap(YamlProtoConfig, newYamlProtoConfig); - Send(subscriber, notification.Release()); + THashSet<ui32> affectedKinds; + for (const auto& kind : ev->Get()->Record.GetAffectedKinds()) { + affectedKinds.insert(kind); } -} -void TConfigsDispatcher::CleanUpSubscriptions() -{ - BLOG_N("Cleaning up all current subscriptions"); - - // If there are active subscriptions then we should - // mark them as removed by reseting their IDs - // and configs. - for (auto &subscription : Subscriptions) { - subscription->SubscriptionId = 0; - subscription->SubscribersToUpdate.clear(); - subscription->UpdateInProcess = nullptr; - subscription->FirstUpdate = true; - } - SubscriptionsById.clear(); - RequestCookies.clear(); - OutOfOrderConfigNotifications.clear(); + for (auto &[kinds, subscription] : SubscriptionsByKinds) { + if (subscription->UpdateInProcess) { + subscription->UpdateInProcess = nullptr; + subscription->SubscribersToUpdate.clear(); + } - // We should invalidate configs cache to avoid its usage until - // updated configs are received. - ConfigsCache.clear(); + NKikimrConfig::TAppConfig trunc; - TDynBitMap kinds; - kinds.Set(NKikimrConsole::TConfigItem::ConfigsDispatcherConfigItem); - auto subscription = FindSubscription(kinds); - if (subscription) { - CreateSubscriberActor(kinds, true); - } else { - AddSubscription(SelfId(), kinds, true); - } + bool hasAffectedKinds = false; - Become(&TThis::StateConfigure); -} + if (subscription->Yaml && YamlConfigEnabled) { + ReplaceConfigItems(YamlProtoConfig, trunc, subscription->Kinds, InitialConfig); + } else { + Y_FOR_EACH_BIT(kind, kinds) { + if (affectedKinds.contains(kind)) { + hasAffectedKinds = true; + } + } -void TConfigsDispatcher::ProcessAddedSubscription(TSubscription::TPtr subscription, ui64 id) -{ - BLOG_N("Confirmed CMS subscription" - << " kinds=" << KindsToString(subscription->Kinds) - << " id=" << id); + // we try resend all configs if yaml config was turned off + if (!hasAffectedKinds && !yamlConfigTurnedOff && CurrentStateFunc() != &TThis::StateInit) { + continue; + } - Y_VERIFY(!subscription->SubscriptionId); - subscription->SubscriptionId = id; - SubscriptionsById[id] = subscription; + ReplaceConfigItems(ev->Get()->Record.GetConfig(), trunc, kinds, InitialConfig); + } - for (auto &subscriber : subscription->Subscribers) { - BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " << subscriber); + if (hasAffectedKinds || !CompareConfigs(subscription->CurrentConfig.Config, trunc) || CurrentStateFunc() == &TThis::StateInit) { + subscription->UpdateInProcess = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); + subscription->UpdateInProcess->Record.MutableConfig()->CopyFrom(trunc); + subscription->UpdateInProcess->Record.SetLocal(true); + Y_FOR_EACH_BIT(kind, kinds) { + subscription->UpdateInProcess->Record.AddItemKinds(kind); + } + subscription->UpdateInProcessCookie = ++NextRequestCookie; + subscription->UpdateInProcessConfigVersion = FilterVersion(ev->Get()->Record.GetConfig().GetVersion(), kinds); - Send(subscriber, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - } + if (YamlConfigEnabled) { + UpdateYamlVersion(subscription); + } - auto it = OutOfOrderConfigNotifications.find(id); - if (it != OutOfOrderConfigNotifications.end()) { - auto ev = std::move(it->second); - OutOfOrderConfigNotifications.erase(it); - Handle(ev); + for (auto &[subscriber, updates] : subscription->Subscribers) { + auto k = kinds; + BLOG_TRACE("Sending for kinds: " << KindsToString(k)); + SendUpdateToSubscriber(subscription, subscriber); + ++updates; + } + } else if (YamlConfigEnabled && subscription->Yaml) { + UpdateYamlVersion(subscription); + } else if (!YamlConfigEnabled) { + subscription->YamlVersion = std::nullopt; + } } - - while (!(SubscriptionsById.size() < Subscriptions.size()) && !OutOfOrderConfigNotifications.empty()) { - auto it = OutOfOrderConfigNotifications.begin(); - auto ev = std::move(it->second); - OutOfOrderConfigNotifications.erase(it); - SendNotificationResponse(ev); + + if (CurrentStateFunc() == &TThis::StateInit) { + Become(&TThis::StateWork); + ProcessEnqueuedEvents(); } - - // Probably there are no more subscribers for this subscription. - // In that case it should be removed. - if (subscription->Subscribers.empty()) - RemoveSubscription(subscription); } -void TConfigsDispatcher::ProcessLocalCacheUpdate(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) +void TConfigsDispatcher::UpdateYamlVersion(const TSubscription::TPtr &subscription) const { - auto &rec = ev->Get()->Record; - BLOG_D("Got new config: " << rec.ShortDebugString()); - - auto subscription = FindSubscription(rec.GetSubscriptionId()); - if (!subscription) { - BLOG_ERROR("Cannot find subscription for configs cache update subscriptionid=" << rec.GetSubscriptionId()); - return; + TYamlVersion yamlVersion; + yamlVersion.Version = NYamlConfig::GetVersion(YamlConfig); + for (auto &[id, hash] : VolatileYamlConfigHashes) { + yamlVersion.VolatileVersions[id] = hash; } - - BLOG_D("Update local cache for kinds=" << KindsToString(subscription->Kinds) - << " config='" << rec.GetConfig().ShortDebugString() << "'"); - - ConfigsCache[subscription->Kinds].reset(new NKikimrConfig::TAppConfig(rec.GetConfig())); - - auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); - - BLOG_TRACE("Send TEvConsole::TEvConfigNotificationResponse to self: " << resp->Record.ShortDebugString()); - - Send(ev->Sender, resp.Release(), 0, ev->Cookie); + subscription->UpdateInProcessYamlVersion = yamlVersion; } -void TConfigsDispatcher::Handle(NMon::TEvHttpInfo::TPtr &ev) +void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionError::TPtr &ev) { - TStringStream str; - str << NMonitoring::HTTPOKHTML; - HTML(str) { - NHttp::OutputStaticPart(str); - PRE() { - str << "Maintained tenant: " << JoinSeq(", ", CurrentTenants); - } - DIV_CLASS("tab-left") { - COLLAPSED_REF_CONTENT("node-labels", "Node labels") { - PRE() { - for (auto& [key, value] : Labels) { - str << key << " = " << value << Endl; - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("current-config", "Current config") { - NHttp::OutputConfigHTML(str, CurrentConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("initial-config", "Initial config") { - NHttp::OutputConfigHTML(str, InitialConfig); - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("subscriptions", "Subscriptions") { - PRE() { - for (auto subscription : Subscriptions) { - str << " - ID: " << subscription->SubscriptionId << Endl - << " Kinds: " << KindsToString(subscription->Kinds) << Endl - << " Subscribers:"; - for (auto &id : subscription->Subscribers) - str << " " << id; - str << Endl - << " FirstUpdate: " << subscription->FirstUpdate << Endl - << " CurrentConfigId: " << subscription->CurrentConfig.ConfigId.ToString() << Endl - << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; - if (subscription->UpdateInProcess) { - str << " UpdateInProcess: " << subscription->UpdateInProcess->Get()->Record.ShortDebugString() << Endl - << " SubscribersToUpdate:"; - for (auto &id : subscription->SubscribersToUpdate) - str << " " << id; - str << Endl; - } - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("subscribers", "Subscribers") { - PRE() { - for (auto &pr : Subscribers) { - str << " - Subscriber: " << pr.second->Subscriber << Endl - << " Subscriptions:"; - for (auto subscription : pr.second->Subscriptions) - str << " " << subscription->SubscriptionId; - str << Endl - << " CurrentConfigId: " << pr.second->CurrentConfigId.ToString() << Endl; - } - } - } - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("cache", "Configs cache") { - DIV_CLASS("tab-left") { - ui32 id = 1; - for (auto &pr : ConfigsCache) { - TString kinds = KindsToString(pr.first); - str << "<br />" << Endl; - COLLAPSED_REF_CONTENT("cache-" + ToString(id++), kinds) { - DIV_CLASS("tab-left") { - PRE() { - str << pr.second->DebugString() << Endl; - } - } - } - } - } - } - } - } - - Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + // The only reason we can get this response is ambiguous domain + // So it is okay to fail here + Y_FAIL("Can't start Configs Dispatcher: %s", + ev->Get()->Record.GetReason().c_str()); } void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvGetConfigRequest::TPtr &ev) { - auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds); + auto resp = MakeHolder<TEvConfigsDispatcher::TEvGetConfigResponse>(); - if (ev->Get()->Cache && !kinds.Empty()) { - auto subscription = FindSubscription(kinds); - if (!subscription || !subscription->Subscribers.contains(SelfId())) { - BLOG_D("Add subscription for local cache kinds=" << KindsToString(kinds)); - AddSubscription(SelfId(), kinds, false); + for (auto kind : ev->Get()->ConfigItemKinds) { + if (!DYNAMIC_KINDS.contains(kind)) { + TStringStream sstr; + sstr << static_cast<NKikimrConsole::TConfigItem::EKind>(kind); + Y_FAIL("unexpected kind in GetConfigRequest: %s", sstr.Str().data()); } } - if (ConfigsCache.contains(kinds)) { - auto resp = MakeHolder<TEvConfigsDispatcher::TEvGetConfigResponse>(); - resp->Config = ConfigsCache.at(kinds); + auto trunc = std::make_shared<NKikimrConfig::TAppConfig>(); + ReplaceConfigItems(CurrentConfig, *trunc, KindsToBitMap(ev->Get()->ConfigItemKinds), InitialConfig); + resp->Config = trunc; - BLOG_TRACE("Send TEvConfigsDispatcher::TEvGetConfigResponse" - " to " << ev->Sender << ": " << resp->Config->ShortDebugString()); + BLOG_TRACE("Send TEvConfigsDispatcher::TEvGetConfigResponse" + " to " << ev->Sender << ": " << resp->Config->ShortDebugString()); - Send(ev->Sender, std::move(resp), 0, ev->Cookie); - } else { - Register(CreateNodeConfigCourier(ev->Get()->ConfigItemKinds, SelfId(), NextRequestCookie)); - ConfigRequests[NextRequestCookie++] = THolder<IEventHandle>(ev.Release()); - } + Send(ev->Sender, std::move(resp), 0, ev->Cookie); } void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest::TPtr &ev) { - auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds); - auto subscriber = FindSubscriber(ev->Sender); - - if (subscriber) { - Y_VERIFY(subscriber->Subscriptions.size() == 1); - auto subscription = *subscriber->Subscriptions.begin(); - - if (subscription->Kinds == kinds) { - BLOG_D("Nothing to change for " << subscriber->Subscriber); - BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " << subscriber->Subscriber); + bool yamlKinds = false; + bool nonYamlKinds = false; + for (auto kind : ev->Get()->ConfigItemKinds) { + if (!DYNAMIC_KINDS.contains(kind)) { + TStringStream sstr; + sstr << static_cast<NKikimrConsole::TConfigItem::EKind>(kind); + Y_FAIL("unexpected kind in SetConfigSubscriptionRequest: %s", sstr.Str().data()); + } - Send(subscriber->Subscriber, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); - return; + if (NON_YAML_KINDS.contains(kind)) { + nonYamlKinds = true; + } else { + yamlKinds = true; } + } - // something changed so refresh subscription - RemoveSubscriber(subscriber); + if (yamlKinds && nonYamlKinds) { + Y_FAIL("both yaml and non yaml kinds in SetConfigSubscriptionRequest"); } - if (!kinds.Empty()) { - AddSubscription(ev->Sender, kinds, false); - } else { - BLOG_TRACE("Send TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse to " << ev->Sender); + auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds); + auto subscriberActor = ev->Get()->Subscriber ? ev->Get()->Subscriber : ev->Sender; - Send(ev->Sender, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + auto subscription = FindSubscription(kinds); + if (!subscription) { + subscription = new TSubscription; + subscription->Kinds = kinds; + subscription->Yaml = yamlKinds; + + SubscriptionsByKinds.emplace(kinds, subscription); + } + auto [subscriberIt, _] = subscription->Subscribers.emplace(subscriberActor, 0); + SubscriptionsBySubscriber.emplace(subscriberActor, subscription); + + auto subscriber = FindSubscriber(subscriberActor); + if (!subscriber) { + subscriber = new TSubscriber; + subscriber->Subscriber = subscriberActor; + Subscribers.emplace(subscriberActor, subscriber); + } + subscriber->Subscriptions.insert(subscription); + + // We don't care about versions and kinds here + Send(ev->Sender, new TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + + if (CurrentStateFunc() != &TThis::StateInit) { + // first time we send even empty config + if (!subscription->UpdateInProcess) { + subscription->UpdateInProcess = MakeHolder<TEvConsole::TEvConfigNotificationRequest>(); + NKikimrConfig::TAppConfig trunc; + if (YamlConfigEnabled) { + ReplaceConfigItems(YamlProtoConfig, trunc, kinds, InitialConfig); + } else { + ReplaceConfigItems(CurrentConfig, trunc, kinds, InitialConfig); + } + subscription->UpdateInProcess->Record.MutableConfig()->CopyFrom(trunc); + Y_FOR_EACH_BIT(kind, kinds) { + subscription->UpdateInProcess->Record.AddItemKinds(kind); + } + subscription->UpdateInProcessCookie = ++NextRequestCookie; + subscription->UpdateInProcessConfigVersion = FilterVersion(CurrentConfig.GetVersion(), kinds); + } + BLOG_TRACE("Sending for kinds: " << KindsToString(kinds)); + SendUpdateToSubscriber(subscription, subscriber->Subscriber); + ++(subscriberIt->second); } } -void TConfigsDispatcher::Handle(TEvConsole::TEvAddConfigSubscriptionResponse::TPtr &ev) +void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest::TPtr &ev) { - auto it = RequestCookies.find(ev->Cookie); - if (it == RequestCookies.end()) { - BLOG_I("Cookie mismatch for TEvAddConfigSubscriptionResponse"); + auto subscriberActor = ev->Get()->Subscriber ? ev->Get()->Subscriber : ev->Sender; + auto subscriber = FindSubscriber(subscriberActor); + if (!subscriber) { return; } - auto kinds = it->second; - RequestCookies.erase(it); - auto &rec = ev->Get()->Record; - if (rec.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::CONFIGS_DISPATCHER, - "Cannot get config subscription for " << KindsToString(kinds) - << " code=" << rec.GetStatus().GetCode() - << " reason= " << rec.GetStatus().GetReason()); - CreateSubscriberActor(kinds, false); - return; + for (auto &subscription : subscriber->Subscriptions) { + subscription->SubscribersToUpdate.erase(subscriberActor); + if (subscription->SubscribersToUpdate.empty()) { + if (subscription->UpdateInProcess) { + subscription->CurrentConfig.Version = subscription->UpdateInProcessConfigVersion; + subscription->CurrentConfig.Config = subscription->UpdateInProcess->Record.GetConfig(); + } + subscription->YamlVersion = subscription->UpdateInProcessYamlVersion; + subscription->UpdateInProcessYamlVersion = std::nullopt; + subscription->UpdateInProcess = nullptr; + } + + subscription->Subscribers.erase(subscriberActor); + if (subscription->Subscribers.empty()) { + SubscriptionsByKinds.erase(subscription->Kinds); + } } - auto subscription = FindSubscription(kinds); - Y_VERIFY(subscription); + Subscribers.erase(subscriberActor); + SubscriptionsBySubscriber.erase(subscriberActor); - ProcessAddedSubscription(subscription, rec.GetSubscriptionId()); + Send(ev->Sender, new TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); } + void TConfigsDispatcher::Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev) { auto rec = ev->Get()->Record; - auto subscription = FindSubscription(rec.GetSubscriptionId()); + auto subscription = FindSubscription(ev->Sender); // Probably subscription was cleared up due to tenant's change. if (!subscription) { - BLOG_I("Got notification response for unknown subscription " << rec.GetSubscriptionId()); + BLOG_ERROR("Got notification response for unknown subscription " << ev->Sender); return; } if (!subscription->UpdateInProcess) { - BLOG_D("Notification was ignored for subscription " - << rec.GetSubscriptionId()); - return; - } - - if (ev->Cookie != subscription->UpdateInProcess->Cookie) { - BLOG_ERROR("Notification cookie mismatch for subscription " << rec.GetSubscriptionId()); + BLOG_D("Notification was ignored for subscription " << ev->Sender); return; } - TConfigId id1(subscription->UpdateInProcess->Get()->Record.GetConfigId()); - TConfigId id2(rec.GetConfigId()); - // This might be outdated notification response. - if (id1 != id2) { - BLOG_I("Config id mismatch in notification response for subscription " << rec.GetSubscriptionId()); + if (ev->Cookie != subscription->UpdateInProcessCookie) { + BLOG_ERROR("Notification cookie mismatch for subscription " << ev->Sender << " " << ev->Cookie << " != " << subscription->UpdateInProcessCookie); + // TODO fix clients return; } if (!subscription->SubscribersToUpdate.contains(ev->Sender)) { - BLOG_ERROR("Notification from unexpected subscriber for subscription " << rec.GetSubscriptionId()); + BLOG_ERROR("Notification from unexpected subscriber for subscription " << ev->Sender); return; } - Subscribers.at(ev->Sender)->CurrentConfigId = id1; + Subscribers.at(ev->Sender)->CurrentConfigVersion = subscription->UpdateInProcessConfigVersion; // If all subscribers responded then send response to CMS. subscription->SubscribersToUpdate.erase(ev->Sender); - MaybeSendNotificationResponse(subscription); -} - -void TConfigsDispatcher::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev) -{ - // Process local update sent by own local subscription. - if (ev->Sender == SelfId()) { - ProcessLocalCacheUpdate(ev); - return; - } - - auto &rec = ev->Get()->Record; - auto subscription = FindSubscription(rec.GetSubscriptionId()); - if (!subscription) { - BLOG_W("Got notification for unknown subscription id=" << rec.GetSubscriptionId()); - - if (SubscriptionsById.size() < Subscriptions.size()) { - // There are subscriptions that don't have an id yet - // Delay processing until we know ids of all subscriptions - auto &prev = OutOfOrderConfigNotifications[rec.GetSubscriptionId()]; - if (prev) { - SendNotificationResponse(prev); - } - prev = ev; - return; - } - - SendNotificationResponse(ev); - return; - } - if (subscription->UpdateInProcess) { - BLOG_D("Drop previous unfinished notification for subscription id=" - << subscription->SubscriptionId); + if (subscription->SubscribersToUpdate.empty()) { + subscription->CurrentConfig.Config = subscription->UpdateInProcess->Record.GetConfig(); + subscription->CurrentConfig.Version = subscription->UpdateInProcessConfigVersion; + subscription->YamlVersion = subscription->UpdateInProcessYamlVersion; + subscription->UpdateInProcessYamlVersion = std::nullopt; subscription->UpdateInProcess = nullptr; - subscription->SubscribersToUpdate.clear(); - } - - subscription->UpdateInProcess = std::move(ev); - - /** - * Avoid notifications in case only config id changed and - * config body is equal to currently used one. - */ - if (subscription->FirstUpdate || !CompareConfigs(subscription->CurrentConfig.Config, rec.GetConfig())) { - for (auto &subscriber : subscription->Subscribers) - SendUpdateToSubscriber(subscription, subscriber); - } else { - MaybeSendNotificationResponse(subscription); } - - subscription->FirstUpdate = false; } -void TConfigsDispatcher::Handle(TEvConsole::TEvGetNodeConfigResponse::TPtr &ev) -{ - auto it = ConfigRequests.find(ev->Cookie); - - if (it == ConfigRequests.end()) { - BLOG_ERROR("Node config response for unknown request cookie=" << ev->Cookie); - return; - } - - auto resp = MakeHolder<TEvConfigsDispatcher::TEvGetConfigResponse>(); - resp->Config.reset(new NKikimrConfig::TAppConfig(ev->Get()->Record.GetConfig())); - BLOG_TRACE("Send TEvConfigsDispatcher::TEvGetConfigResponse" - " to " << ev->Sender - << ": " << resp->Config->ShortDebugString()); +void TConfigsDispatcher::Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev) { + auto Response = MakeHolder<TEvConsole::TEvGetNodeLabelsResponse>(); - Send(it->second->Sender, resp.Release(), 0, it->second->Cookie); - - ConfigRequests.erase(it); -} - -void TConfigsDispatcher::Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev) -{ - auto it = RequestCookies.find(ev->Cookie); - if (it == RequestCookies.end()) { - BLOG_ERROR("Cookie mismatch for TEvReplaceConfigSubscriptionsResponse"); - return; - } - - auto &rec = ev->Get()->Record; - if (rec.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::CONFIGS_DISPATCHER, - "Cannot initialize subscription: " << rec.GetStatus().GetReason()); - CleanUpSubscriptions(); - return; + for (const auto& [label, value] : Labels) { + auto *labelSer = Response->Record.MutableResponse()->add_labels(); + labelSer->set_label(label); + labelSer->set_value(value); } - auto subscription = FindSubscription(it->second); - Y_VERIFY(subscription); - ProcessAddedSubscription(subscription, rec.GetSubscriptionId()); - - // Register other subscriptions in CMS. - for (auto subscription : Subscriptions) - if (!subscription->SubscriptionId) - CreateSubscriberActor(subscription->Kinds, false); - - Become(&TThis::StateWork); - ProcessEnqueuedEvents(); -} - -void TConfigsDispatcher::Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev) -{ - auto &rec = ev->Get()->Record; - - THashSet<TString> tenants; - for (auto &slot : rec.GetSlots()) - tenants.insert(slot.GetAssignedTenant()); - - // If we are in initial state then subscriptions set is empty - // and we should start initialization. Otherwise we should - // re-initialize if tenants set changed. - if (CurrentTenants != tenants || Subscriptions.empty()) { - CurrentTenants = tenants; - - BLOG_N("Update list of assigned tenants: " << JoinSeq(", ", CurrentTenants)); - - CleanUpSubscriptions(); - } + Send(ev->Sender, Response.Release()); } - -} // anonymous namespace - -IActor *CreateConfigsDispatcher(const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels) + +IActor *CreateConfigsDispatcher( + const NKikimrConfig::TAppConfig &config, + const TMap<TString, TString> &labels, + const NKikimrConfig::TAppConfig &initialCmsConfig, + const NKikimrConfig::TAppConfig &initialCmsYamlConfig) { - return new TConfigsDispatcher(config, labels); + return new TConfigsDispatcher(config, labels, initialCmsConfig, initialCmsYamlConfig); } } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/configs_dispatcher.h b/ydb/core/cms/console/configs_dispatcher.h index a1511eecd6..879e8bf58f 100644 --- a/ydb/core/cms/console/configs_dispatcher.h +++ b/ydb/core/cms/console/configs_dispatcher.h @@ -32,6 +32,8 @@ struct TEvConfigsDispatcher { EvSetConfigSubscriptionResponse, EvGetConfigRequest, EvGetConfigResponse, + EvRemoveConfigSubscriptionRequest, + EvRemoveConfigSubscriptionResponse, EvEnd }; @@ -40,26 +42,48 @@ struct TEvConfigsDispatcher { "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_CONFIGS_DISPATCHER)"); struct TEvSetConfigSubscriptionRequest : public TEventLocal<TEvSetConfigSubscriptionRequest, EvSetConfigSubscriptionRequest> { - TEvSetConfigSubscriptionRequest() + TEvSetConfigSubscriptionRequest(TActorId subscriber = {}) + : Subscriber(subscriber) { } - TEvSetConfigSubscriptionRequest(ui32 kind) + TEvSetConfigSubscriptionRequest(ui32 kind, TActorId subscriber = {}) : ConfigItemKinds({kind}) + , Subscriber(subscriber) { } - TEvSetConfigSubscriptionRequest(std::initializer_list<ui32> kinds) + TEvSetConfigSubscriptionRequest(std::initializer_list<ui32> kinds, TActorId subscriber = {}) : ConfigItemKinds(kinds) + , Subscriber(subscriber) + { + } + + TEvSetConfigSubscriptionRequest(const TVector<ui32> &kinds, TActorId subscriber = {}) + : ConfigItemKinds(kinds) + , Subscriber(subscriber) { } TVector<ui32> ConfigItemKinds; + const TActorId Subscriber; }; struct TEvSetConfigSubscriptionResponse : public TEventLocal<TEvSetConfigSubscriptionResponse, EvSetConfigSubscriptionResponse> { }; + struct TEvRemoveConfigSubscriptionRequest : public TEventLocal<TEvRemoveConfigSubscriptionRequest, EvRemoveConfigSubscriptionRequest> { + TEvRemoveConfigSubscriptionRequest(TActorId subscriber = {}) + : Subscriber(subscriber) + { + } + + const TActorId Subscriber; + }; + + struct TEvRemoveConfigSubscriptionResponse : public TEventLocal<TEvRemoveConfigSubscriptionResponse, EvRemoveConfigSubscriptionResponse> { + }; + struct TEvGetConfigRequest : public TEventLocal<TEvGetConfigRequest, EvGetConfigRequest> { TEvGetConfigRequest(ui32 kind, bool cache = true) : ConfigItemKinds({kind}) @@ -87,7 +111,11 @@ struct TEvConfigsDispatcher { * are compared to the current one and notifications are not sent to local * subscribers if there is no config modification detected. */ -IActor *CreateConfigsDispatcher(const NKikimrConfig::TAppConfig &config, const TMap<TString, TString> &labels); +IActor *CreateConfigsDispatcher( + const NKikimrConfig::TAppConfig &config, + const TMap<TString, TString> &labels, + const NKikimrConfig::TAppConfig &initialCmsConfig = {}, + const NKikimrConfig::TAppConfig &initialCmsYamlConfig = {}); inline TActorId MakeConfigsDispatcherID(ui32 node = 0) { char x[12] = { 'c', 'o', 'n', 'f', 'i', 'g', 's', 'd', 'i', 's', 'p' }; diff --git a/ydb/core/cms/console/configs_dispatcher_ut.cpp b/ydb/core/cms/console/configs_dispatcher_ut.cpp index 8c9a26890d..593e4b759c 100644 --- a/ydb/core/cms/console/configs_dispatcher_ut.cpp +++ b/ydb/core/cms/console/configs_dispatcher_ut.cpp @@ -55,6 +55,9 @@ TTenantTestConfig DefaultConsoleTestConfig() NKikimrConsole::TConfigItem ITEM_DOMAIN_LOG_1; NKikimrConsole::TConfigItem ITEM_DOMAIN_LOG_2; +NKikimrConsole::TConfigItem ITEM_NET_CLASSIFIER_1; +NKikimrConsole::TConfigItem ITEM_NET_CLASSIFIER_2; +NKikimrConsole::TConfigItem ITEM_NET_CLASSIFIER_3; TActorId InitConfigsDispatcher(TTenantTestRuntime &runtime) { @@ -67,6 +70,19 @@ TActorId InitConfigsDispatcher(TTenantTestRuntime &runtime) NKikimrConfig::TAppConfig(), {}, {}, "", "", 2, NKikimrConsole::TConfigItem::MERGE, ""); + ITEM_NET_CLASSIFIER_1 + = MakeConfigItem(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, + NKikimrConfig::TAppConfig(), {}, {}, "", "", 3, + NKikimrConsole::TConfigItem::MERGE, ""); + ITEM_NET_CLASSIFIER_2 + = MakeConfigItem(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, + NKikimrConfig::TAppConfig(), {}, {}, "", "", 4, + NKikimrConsole::TConfigItem::MERGE, ""); + ITEM_NET_CLASSIFIER_3 + = MakeConfigItem(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem, + NKikimrConfig::TAppConfig(), {}, {}, "", "", 5, + NKikimrConsole::TConfigItem::MERGE, ""); + return MakeConfigsDispatcherID(runtime.GetNodeId(0)); } @@ -131,6 +147,7 @@ struct TEvPrivate { struct TEvGotNotification : public TEventLocal<TEvGotNotification, EvGotNotification> { TConfigId ConfigId; + NKikimrConfig::TAppConfig Config; }; struct TEvComplete : public TEventLocal<TEvComplete, EvComplete> {}; @@ -187,6 +204,7 @@ public: if (Sink) { auto *event = new TEvPrivate::TEvGotNotification; event->ConfigId.Load(rec.GetConfigId()); + event->Config = rec.GetConfig(); ctx.Send(Sink, event); } @@ -242,15 +260,6 @@ TActorId AddSubscriber(TTenantTestRuntime &runtime, TVector<ui32> kinds, bool ho return aid; } -NKikimrConfig::TAppConfig GetConfig(TTenantTestRuntime &runtime, TVector<ui32> kinds, bool cache = true) -{ - TAutoPtr<IEventHandle> handle; - runtime.Send(new IEventHandle(MakeConfigsDispatcherID(runtime.GetNodeId(0)), - runtime.Sender, - new TEvConfigsDispatcher::TEvGetConfigRequest(kinds, cache))); - return *runtime.GrabEdgeEventRethrow<TEvConfigsDispatcher::TEvGetConfigResponse>(handle)->Config; -} - void HoldSubscriber(TTenantTestRuntime &runtime, TActorId aid) { TAutoPtr<IEventHandle> handle; @@ -271,21 +280,6 @@ void SetSubscriptions(TTenantTestRuntime &runtime, TActorId aid, TVector<ui32> k } // anonymous namespace Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { - Y_UNIT_TEST(TestSelfSubscription) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig()); - auto serviceId = InitConfigsDispatcher(runtime); - - ui64 id; - TDispatchOptions options; - options.FinalEvents.emplace_back(CatchReplaceConfigResult(id), 1); - runtime.DispatchEvents(options); - - CheckListConfigSubscriptions(runtime, Ydb::StatusIds::SUCCESS, 0, serviceId, - id, runtime.GetNodeId(0), FQDNHostName(), TENANT1_1_NAME, "type1", - 0, serviceId, - TVector<ui32>({(ui32)NKikimrConsole::TConfigItem::ConfigsDispatcherConfigItem})); - } - Y_UNIT_TEST(TestSubscriptionNotification) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); TAutoPtr<IEventHandle> handle; @@ -300,11 +294,11 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_1)); - // Expect two responses from subscribers and one from dispatcher. + // Expect two responses from subscribers and zero from dispatcher TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 3); + options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 2); runtime.DispatchEvents(options); - } + } Y_UNIT_TEST(TestSubscriptionNotificationForNewSubscriberAfterUpdate) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); @@ -323,9 +317,9 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { UnholdSubscriber(runtime, s1); - // Expect response from subscriber and from dispatcher. + // Expect response from subscriber TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 2); + options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); runtime.DispatchEvents(options); // New subscriber should get notification. @@ -359,9 +353,9 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { UnholdSubscriber(runtime, s1); - // Expect response from unhold subscriber and from dispatcher. + // Expect response from unhold subscriber TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 2); + options.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); runtime.DispatchEvents(options); } @@ -377,7 +371,6 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { SetSubscriptions(runtime, s1, {}); TDispatchOptions options; - options.FinalEvents.emplace_back(TEvConsole::EvRemoveConfigSubscriptionResponse, 1); runtime.DispatchEvents(options); runtime.GrabEdgeEventRethrow<TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse>(handle); @@ -401,81 +394,183 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { options1.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); runtime.DispatchEvents(options1); - // Subscriber removal should cause config notification response. + // We don't track acks from config dispatcher with InMemory subscriptions SetSubscriptions(runtime, s1, {}); TDispatchOptions options2; - options2.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); runtime.DispatchEvents(options2); } - Y_UNIT_TEST(TestGetCachedConfig) { + Y_UNIT_TEST(TestEmptyChangeCausesNoNotification) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); TAutoPtr<IEventHandle> handle; InitConfigsDispatcher(runtime); - ui64 nodeConfigRequests = 0; - auto observer = [&nodeConfigRequests](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { - switch (ev->GetTypeRewrite()) { - case TEvConsole::EvGetNodeConfigRequest: - ++nodeConfigRequests; - break; + ui64 notifications = 0; + TActorId subscriber; + auto observer = [¬ifications,&subscriber,recipient=runtime.Sender](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + if (ev->Recipient == recipient && ev->Sender == subscriber) { + switch (ev->GetTypeRewrite()) { + case TEvPrivate::EvGotNotification: + ++notifications; + break; + } } return TTestActorRuntime::EEventAction::PROCESS; }; ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); - ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster2"); + ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, MakeAddAction(ITEM_DOMAIN_LOG_1)); + runtime.SetObserverFunc(observer); + + // Add subscriber and get config via notification. + subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); + runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + UNIT_ASSERT(notifications > 0); + + // Now add another element which doesn't change config body. + // It should cause notification to dispatcher but not test subscriber. + SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_2)); + notifications = 0; + TDispatchOptions options1; + runtime.DispatchEvents(options1); + UNIT_ASSERT_VALUES_EQUAL(notifications, 0); + } + + Y_UNIT_TEST(TestYamlAndNonYamlCoexist) { NKikimrConfig::TAppConfig config; - config.MutableLogConfig()->SetClusterName("cluster1"); + auto *label = config.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), config); + TAutoPtr<IEventHandle> handle; + InitConfigsDispatcher(runtime); + + ui64 notifications = 0; + TActorId subscriber; + auto observer = [¬ifications, &subscriber, recipient = runtime.Sender]( + TTestActorRuntimeBase&, + TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + if (ev->Recipient == recipient && ev->Sender == subscriber) { + switch (ev->GetTypeRewrite()) { + case TEvPrivate::EvGotNotification: + ++notifications; + break; + } + } + return TTestActorRuntime::EEventAction::PROCESS; + }; runtime.SetObserverFunc(observer); - // Config should be requested from CMS. - auto config1 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, false); - CheckEqualsIgnoringVersion(config, config1); - UNIT_ASSERT(nodeConfigRequests > 0); + ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); + ITEM_NET_CLASSIFIER_1.MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateTimestamp(1); + + CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, + MakeAddAction(ITEM_DOMAIN_LOG_1), + MakeAddAction(ITEM_NET_CLASSIFIER_1)); + + subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem}); + + auto reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + NKikimrConfig::TAppConfig expectedConfig; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + auto *ncdConfig = expectedConfig.MutableNetClassifierDistributableConfig(); + ncdConfig->SetLastUpdateTimestamp(1); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + notifications = 0; - // We didn't ask to cache, so config should still be requested from CMS. - // This time ask to cache config. - nodeConfigRequests = 0; - auto config2 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, true); - CheckEqualsIgnoringVersion(config, config2); - UNIT_ASSERT(nodeConfigRequests > 0); + TString yamlConfig1 = R"( +--- +metadata: + cluster: "" + version: 0 - // Make sure subscription is online by using it with another subscriber. - AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); - runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); +config: + log_config: + cluster_name: cluster2 + net_classifier_distributable_config: + last_update_timestamp: 3 + yaml_config_enabled: true - // This time we should get config with no requests to CMS. - nodeConfigRequests = 0; - auto config3 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, true); - CheckEqualsIgnoringVersion(config, config3); - UNIT_ASSERT_VALUES_EQUAL(nodeConfigRequests, 0); +allowed_labels: + test: + type: enum + values: + ? true - // Change config and expect dispatcher to process notification. - SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_2)); - runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); +selector_config: [] +)"; - // Now we should get new config with no requests to CMS. - config.MutableLogConfig()->SetClusterName("cluster2"); - auto config4 = GetConfig(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}, true); - CheckEqualsIgnoringVersion(config, config4); - UNIT_ASSERT_VALUES_EQUAL(nodeConfigRequests, 0); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig1); + + UNIT_ASSERT(notifications == 0); + + ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster3"); + + CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, + MakeAddAction(ITEM_DOMAIN_LOG_2)); + + UNIT_ASSERT(notifications == 0); + + ITEM_NET_CLASSIFIER_2.MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateTimestamp(3); + + CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, + MakeAddAction(ITEM_NET_CLASSIFIER_2)); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + ncdConfig->SetLastUpdateTimestamp(3); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + notifications = 0; + + TString yamlConfig2 = R"( +--- +metadata: + cluster: "" + version: 1 + +config: {yaml_config_enabled: false} +allowed_labels: {} +selector_config: [] +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig2); + + UNIT_ASSERT(notifications == 0); + + ITEM_NET_CLASSIFIER_3.MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateTimestamp(5); + + CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, + MakeAddAction(ITEM_NET_CLASSIFIER_3)); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + ncdConfig->SetLastUpdateTimestamp(5); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); } - Y_UNIT_TEST(TestEmptyChangeCausesNoNotification) { - TTenantTestRuntime runtime(DefaultConsoleTestConfig()); + Y_UNIT_TEST(TestYamlEndToEnd) { + NKikimrConfig::TAppConfig config; + auto *label = config.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + + TTenantTestRuntime runtime(DefaultConsoleTestConfig(), config); TAutoPtr<IEventHandle> handle; InitConfigsDispatcher(runtime); ui64 notifications = 0; TActorId subscriber; - auto observer = [¬ifications,&subscriber,recipient=runtime.Sender](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + auto observer = [¬ifications, &subscriber, recipient = runtime.Sender]( + TTestActorRuntimeBase&, + TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { if (ev->Recipient == recipient && ev->Sender == subscriber) { switch (ev->GetTypeRewrite()) { case TEvPrivate::EvGotNotification: @@ -485,28 +580,309 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { } return TTestActorRuntime::EEventAction::PROCESS; }; + runtime.SetObserverFunc(observer); ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); - ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster1"); + ITEM_DOMAIN_LOG_1.MutableConfig()->MutableLogConfig()->SetDefaultLevel(5); CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, MakeAddAction(ITEM_DOMAIN_LOG_1)); - runtime.SetObserverFunc(observer); - - // Add subscriber and get config via notification. subscriber = AddSubscriber(runtime, {(ui32)NKikimrConsole::TConfigItem::LogConfigItem}); - runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + auto reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + NKikimrConfig::TAppConfig expectedConfig; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + auto *logConfig = expectedConfig.MutableLogConfig(); + logConfig->SetClusterName("cluster1"); + logConfig->SetDefaultLevel(5); UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + notifications = 0; - // Now add another element which doesn't change config body. - // It should cause notification to dispatcher but not test subscriber. - SendConfigure(runtime, MakeAddAction(ITEM_DOMAIN_LOG_2)); + TString yamlConfig1 = R"( +--- +metadata: + cluster: "" + version: 0 + +config: + log_config: + cluster_name: cluster1 +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig1); + UNIT_ASSERT(notifications == 0); + + TString yamlConfig2 = R"( +--- +metadata: + cluster: "" + version: 1 + +config: + log_config: + cluster_name: cluster1 + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig2); + + ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster2"); + ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetDefaultLevel(5); + + CheckConfigure(runtime, Ydb::StatusIds::SUCCESS, + MakeAddAction(ITEM_DOMAIN_LOG_2)); + + TString yamlConfig3 = R"( +--- +metadata: + cluster: "" + version: 2 + +config: + log_config: + cluster_name: cluster3 + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig3); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + expectedConfig = {}; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + logConfig = expectedConfig.MutableLogConfig(); + logConfig->SetClusterName("cluster3"); + logConfig->SetDefaultLevel(5); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); notifications = 0; - TDispatchOptions options1; - options1.FinalEvents.emplace_back(TEvConsole::EvConfigNotificationResponse, 1); - runtime.DispatchEvents(options1); - UNIT_ASSERT_VALUES_EQUAL(notifications, 0); + + TString yamlConfig4 = R"( +--- +metadata: + cluster: "" + version: 3 + +config: + log_config: + cluster_name: cluster3 + cms_config: + sentinel_config: + enable: true + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig4); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig4); + UNIT_ASSERT(notifications == 0); + + TString yamlConfig5 = R"( +--- +metadata: + cluster: "" + version: 4 + +config: + log_config: + cluster_name: cluster3 + cms_config: + sentinel_config: + enable: true + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: +- description: Test + selector: + test: true + config: + log_config: !inherit + entry: + - component: AUDIT_LOG_WRITER + level: 7 +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig5); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + expectedConfig = {}; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + logConfig = expectedConfig.MutableLogConfig(); + logConfig->SetClusterName("cluster3"); + logConfig->SetDefaultLevel(5); + auto *entry = logConfig->AddEntry(); + entry->SetComponent("AUDIT_LOG_WRITER"); + entry->SetLevel(7); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + notifications = 0; + + TString yamlConfig6 = R"( +--- +metadata: + cluster: "" + version: 5 + +config: + log_config: + cluster_name: cluster3 + cms_config: + sentinel_config: + enable: true + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: +- description: Test + selector: + test: true + config: + log_config: !inherit + entry: + - component: AUDIT_LOG_WRITER + level: 6 +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig6); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + expectedConfig = {}; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + logConfig = expectedConfig.MutableLogConfig(); + logConfig->SetClusterName("cluster3"); + logConfig->SetDefaultLevel(5); + entry = logConfig->AddEntry(); + entry->SetComponent("AUDIT_LOG_WRITER"); + entry->SetLevel(6); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + notifications = 0; + + TString yamlConfig7 = R"( +--- +metadata: + cluster: "" + version: 6 +config: + log_config: + cluster_name: cluster3 + cms_config: + sentinel_config: + enable: true + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: +- description: Test + selector: + test: + not_in: + - true + config: + log_config: !inherit + entry: + - component: AUDIT_LOG_WRITER + level: 7 +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig7); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + expectedConfig = {}; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + logConfig = expectedConfig.MutableLogConfig(); + logConfig->SetClusterName("cluster3"); + logConfig->SetDefaultLevel(5); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); + notifications = 0; + + TString yamlConfig8 = R"( +--- +metadata: + cluster: "" + version: 7 + +config: + log_config: + cluster_name: cluster3 + yaml_config_enabled: true + +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: +- description: Test + selector: + test: true + config: + yaml_config_enabled: false +)"; + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, yamlConfig8); + + reply = runtime.GrabEdgeEventRethrow<TEvPrivate::TEvGotNotification>(handle); + expectedConfig = {}; + label = expectedConfig.AddLabels(); + label->SetName("test"); + label->SetValue("true"); + logConfig = expectedConfig.MutableLogConfig(); + logConfig->SetClusterName("cluster2"); + logConfig->SetDefaultLevel(5); + UNIT_ASSERT(notifications > 0); + UNIT_ASSERT_VALUES_EQUAL(expectedConfig.ShortDebugString(), reply->Config.ShortDebugString()); } } diff --git a/ydb/core/cms/console/console.h b/ydb/core/cms/console/console.h index 5a8716bba1..53997b63bc 100644 --- a/ydb/core/cms/console/console.h +++ b/ydb/core/cms/console/console.h @@ -8,7 +8,7 @@ #include <ydb/core/protos/console_config.pb.h> #include <ydb/core/protos/console_tenant.pb.h> #include <ydb/public/api/protos/ydb_cms.pb.h> -#include <ydb/public/api/protos/draft/ydb_console.pb.h> +#include <ydb/public/api/protos/draft/ydb_dynamic_config.pb.h> namespace NKikimr::NConsole { @@ -45,12 +45,16 @@ struct TEvConsole { EvUpdateTenantPoolConfig, EvGetLogTailRequest, // - EvApplyConfigRequest, + EvSetYamlConfigRequest, EvAddVolatileConfigRequest, EvRemoveVolatileConfigRequest, EvGetAllConfigsRequest, EvResolveConfigRequest, EvResolveAllConfigRequest, + EvDropConfigRequest, + EvReplaceYamlConfigRequest, + EvGetAllMetadataRequest, + EvGetNodeLabelsRequest, // responses EvCreateTenantResponse = EvCreateTenantRequest + 1024, @@ -82,12 +86,19 @@ struct TEvConsole { EvConfigSubscriptionError, EvGetLogTailResponse, // - EvApplyConfigResponse, + EvSetYamlConfigResponse, EvAddVolatileConfigResponse, EvRemoveVolatileConfigResponse, EvGetAllConfigsResponse, EvResolveConfigResponse, EvResolveAllConfigResponse, + EvDropConfigResponse, + EvReplaceYamlConfigResponse, + EvGetAllMetadataResponse, + EvGetNodeLabelsResponse, + EvUnauthorized, + EvDisabled, + EvGenericError, EvEnd }; @@ -149,10 +160,22 @@ struct TEvConsole { ////////////////////////////////////////////////// // NEW CONFIGS MANAGEMENT ////////////////////////////////////////////////// - struct TEvApplyConfigResponse : public TEventShortDebugPB<TEvApplyConfigResponse, NKikimrConsole::TApplyConfigResponse, EvApplyConfigResponse> {}; + struct TEvSetYamlConfigResponse : public TEventShortDebugPB<TEvSetYamlConfigResponse, NKikimrConsole::TSetYamlConfigResponse, EvSetYamlConfigResponse> {}; - struct TEvApplyConfigRequest : public TEventShortDebugPB<TEvApplyConfigRequest, NKikimrConsole::TApplyConfigRequest, EvApplyConfigRequest> { - using TResponse = TEvApplyConfigResponse; + struct TEvSetYamlConfigRequest : public TEventShortDebugPB<TEvSetYamlConfigRequest, NKikimrConsole::TSetYamlConfigRequest, EvSetYamlConfigRequest> { + using TResponse = TEvSetYamlConfigResponse; + }; + + struct TEvReplaceYamlConfigResponse : public TEventShortDebugPB<TEvReplaceYamlConfigResponse, NKikimrConsole::TReplaceYamlConfigResponse, EvReplaceYamlConfigResponse> {}; + + struct TEvReplaceYamlConfigRequest : public TEventShortDebugPB<TEvReplaceYamlConfigRequest, NKikimrConsole::TReplaceYamlConfigRequest, EvReplaceYamlConfigRequest> { + using TResponse = TEvReplaceYamlConfigResponse; + }; + + struct TEvDropConfigResponse : public TEventShortDebugPB<TEvDropConfigResponse, NKikimrConsole::TDropConfigResponse, EvDropConfigResponse> {}; + + struct TEvDropConfigRequest : public TEventShortDebugPB<TEvDropConfigRequest, NKikimrConsole::TDropConfigRequest, EvDropConfigRequest> { + using TResponse = TEvDropConfigResponse; }; struct TEvAddVolatileConfigResponse : public TEventShortDebugPB<TEvAddVolatileConfigResponse, NKikimrConsole::TAddVolatileConfigResponse, EvAddVolatileConfigResponse> {}; @@ -173,6 +196,18 @@ struct TEvConsole { using TResponse = TEvGetAllConfigsResponse; }; + struct TEvGetAllMetadataResponse : public TEventShortDebugPB<TEvGetAllMetadataResponse, NKikimrConsole::TGetAllMetadataResponse, EvGetAllMetadataResponse> {}; + + struct TEvGetAllMetadataRequest : public TEventShortDebugPB<TEvGetAllMetadataRequest, NKikimrConsole::TGetAllMetadataRequest, EvGetAllMetadataRequest> { + using TResponse = TEvGetAllMetadataResponse; + }; + + struct TEvGetNodeLabelsResponse : public TEventShortDebugPB<TEvGetNodeLabelsResponse, NKikimrConsole::TGetNodeLabelsResponse, EvGetNodeLabelsResponse> {}; + + struct TEvGetNodeLabelsRequest : public TEventShortDebugPB<TEvGetNodeLabelsRequest, NKikimrConsole::TGetNodeLabelsRequest, EvGetNodeLabelsRequest> { + using TResponse = TEvGetNodeLabelsResponse; + }; + struct TEvResolveConfigRequest : public TEventShortDebugPB<TEvResolveConfigRequest, NKikimrConsole::TResolveConfigRequest, EvResolveConfigRequest> {}; struct TEvResolveConfigResponse : public TEventShortDebugPB<TEvResolveConfigResponse, NKikimrConsole::TResolveConfigResponse, EvResolveConfigResponse> {}; @@ -181,6 +216,12 @@ struct TEvConsole { struct TEvResolveAllConfigResponse : public TEventShortDebugPB<TEvResolveAllConfigResponse, NKikimrConsole::TResolveAllConfigResponse, EvResolveAllConfigResponse> {}; + struct TEvUnauthorized : public TEventShortDebugPB<TEvUnauthorized, NKikimrConsole::TUnauthorized, EvUnauthorized> {}; + + struct TEvDisabled : public TEventShortDebugPB<TEvDisabled, NKikimrConsole::TDisabled, EvDisabled> {}; + + struct TEvGenericError : public TEventShortDebugPB<TEvGenericError, NKikimrConsole::TGenericError, EvGenericError> {}; + ////////////////////////////////////////////////// // CMS MANAGEMENT ////////////////////////////////////////////////// diff --git a/ydb/core/cms/console/console__apply_yaml_config.cpp b/ydb/core/cms/console/console__apply_yaml_config.cpp deleted file mode 100644 index 190305e8b4..0000000000 --- a/ydb/core/cms/console/console__apply_yaml_config.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include "console_configs_manager.h" -#include "console_configs_provider.h" - -#include <ydb/core/tablet_flat/tablet_flat_executed.h> - -namespace NKikimr::NConsole { - -using namespace NKikimrConsole; - -class TConfigsManager::TTxApplyYamlConfig : public TTransactionBase<TConfigsManager> { -public: - TTxApplyYamlConfig(TConfigsManager *self, - TEvConsole::TEvApplyConfigRequest::TPtr &ev) - : TBase(self) - , Request(std::move(ev)) - { - } - - bool Execute(TTransactionContext &txc, const TActorContext &) override - { - auto &req = Request->Get()->Record; - - NIceDb::TNiceDb db(txc.DB); - - auto config = req.GetRequest().config(); - - if (config != Self->YamlConfig) { - Modify = true; - - try { - auto parser = NFyaml::TParser::Create(config.Detach()); - auto header = parser.NextDocument(); - auto cluster = header->Root().Map()["cluster"].Scalar(); - Version = TryFromString<ui64>(header->Root().Map()["version"].Scalar()) - .GetOrElse(0); - - if (Version == 0) { - ythrow yexception() << "Invalid version"; - } - - auto tree = parser.NextDocument(); - auto resolved = NYamlConfig::ResolveAll(*tree); - - if (Self->ClusterName != cluster) { - ythrow yexception() << "ClusterName mismatch"; - } - - if (Version != Self->YamlVersion + 1) { - ythrow yexception() << "Version mismatch"; - } - - for (auto& [_, config] : resolved.Configs) { - auto cfg = NYamlConfig::YamlToProto(config.second); - } - } catch (const yexception& ex) { - Response = MakeHolder<TEvConsole::TEvApplyConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::BAD_REQUEST); - op->set_ready(true); - auto *issue = op->add_issues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(ex.what()); - Error = true; - return true; - } - - db.Table<Schema::YamlConfig>().Key(Version) - .Update<Schema::YamlConfig::Config>(config); - - /* Later we shift this boundary to support rollback and history */ - db.Table<Schema::YamlConfig>().Key(Version - 1) - .Delete(); - } - - Response = MakeHolder<TEvConsole::TEvApplyConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); - - return true; - } - - void Complete(const TActorContext &ctx) override - { - LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxApplyYamlConfig Complete"); - - auto &req = Request->Get()->Record; - - ctx.Send(Request->Sender, Response.Release()); - - if (!Error && Modify) { - Self->YamlVersion = Version; - Self->YamlConfig = req.GetRequest().config(); - - Self->VolatileYamlConfigs.clear(); - - auto resp = MakeHolder<TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig>(Self->YamlConfig); - ctx.Send(Self->ConfigsProvider, resp.Release()); - } - - Self->TxProcessor->TxCompleted(this, ctx); - } - -private: - TEvConsole::TEvApplyConfigRequest::TPtr Request; - THolder<TEvConsole::TEvApplyConfigResponse> Response; - bool Error = false; - bool Modify = false; - ui32 Version; -}; - -ITransaction *TConfigsManager::CreateTxApplyYamlConfig(TEvConsole::TEvApplyConfigRequest::TPtr &ev) -{ - return new TTxApplyYamlConfig(this, ev); -} - -} // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console__cleanup_subscriptions.cpp b/ydb/core/cms/console/console__cleanup_subscriptions.cpp index 4a44271a9f..f56c0a74f2 100644 --- a/ydb/core/cms/console/console__cleanup_subscriptions.cpp +++ b/ydb/core/cms/console/console__cleanup_subscriptions.cpp @@ -1,9 +1,9 @@ -#include "config_index.h" #include "console_impl.h" #include "console_configs_manager.h" #include "console_tenants_manager.h" #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/cms/console/validators/registry.h> namespace NKikimr::NConsole { diff --git a/ydb/core/cms/console/console__drop_yaml_config.cpp b/ydb/core/cms/console/console__drop_yaml_config.cpp new file mode 100644 index 0000000000..080fee47a3 --- /dev/null +++ b/ydb/core/cms/console/console__drop_yaml_config.cpp @@ -0,0 +1,96 @@ +#include "console_configs_manager.h" +#include "console_configs_provider.h" + +#include <ydb/core/tablet_flat/tablet_flat_executed.h> + +namespace NKikimr::NConsole { + +using namespace NKikimrConsole; + +class TConfigsManager::TTxDropYamlConfig : public TTransactionBase<TConfigsManager> { +public: + TTxDropYamlConfig(TConfigsManager *self, + TEvConsole::TEvDropConfigRequest::TPtr &ev) + : TBase(self) + , Request(std::move(ev)) + { + } + + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override + { + auto &req = Request->Get()->Record; + + NIceDb::TNiceDb db(txc.DB); + + try { + Version = req.GetRequest().identity().version(); + auto cluster = req.GetRequest().identity().cluster(); + + if (Version == 0) { + ythrow yexception() << "Invalid version"; + } + + if (Self->ClusterName != cluster) { + ythrow yexception() << "ClusterName mismatch"; + } + + if (Version != Self->YamlVersion) { + ythrow yexception() << "Version mismatch"; + } + } catch (const yexception& ex) { + Error = true; + + auto ev = MakeHolder<TEvConsole::TEvGenericError>(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ex.what()); + Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, ev.Release()); + return true; + } + + if (!Self->YamlDropped) { + Modify = true; + + db.Table<Schema::YamlConfig>().Key(Version).Delete(); + } + + Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, new TEvConsole::TEvDropConfigResponse()); + + return true; + } + + void Complete(const TActorContext &ctx) override + { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxDropYamlConfig Complete"); + + ctx.Send(Response.Release()); + + if (!Error && Modify) { + Self->YamlVersion = 0; + Self->YamlConfig.clear(); + Self->YamlDropped = true; + + Self->VolatileYamlConfigs.clear(); + + auto resp = MakeHolder<TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig>(""); + ctx.Send(Self->ConfigsProvider, resp.Release()); + } + + Self->TxProcessor->TxCompleted(this, ctx); + } + +private: + TEvConsole::TEvDropConfigRequest::TPtr Request; + THolder<NActors::IEventHandle> Response; + bool Error = false; + bool Modify = false; + ui32 Version; +}; + +ITransaction *TConfigsManager::CreateTxDropYamlConfig(TEvConsole::TEvDropConfigRequest::TPtr &ev) +{ + return new TTxDropYamlConfig(this, ev); +} + +} // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index 62800326df..d25d264c48 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -9,35 +9,19 @@ using namespace NKikimrConsole; class TConfigsManager::TTxGetYamlConfig : public TTransactionBase<TConfigsManager> { public: TTxGetYamlConfig(TConfigsManager *self, - TEvConsole::TEvGetAllConfigsRequest::TPtr &ev) + TEvConsole::TEvGetAllConfigsRequest::TPtr &ev) : TBase(self) , Request(std::move(ev)) { } - bool Execute(TTransactionContext &txc, const TActorContext &) override + bool Execute(TTransactionContext &, const TActorContext &) override { - NIceDb::TNiceDb db(txc.DB); - - auto rowset = db.Table<Schema::YamlConfig>() - .Reverse() - .Select<Schema::YamlConfig::TColumns>(); - - if (!rowset.IsReady()) - return false; - Response = MakeHolder<TEvConsole::TEvGetAllConfigsResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); - Response->Record.MutableResponse()->set_cluster(Self->ClusterName); - Response->Record.MutableResponse()->set_version(Self->YamlVersion); - - if (!rowset.EndOfSet()) { - auto config = rowset.template GetValue<Schema::YamlConfig::Config>(); - Response->Record.MutableResponse()->set_config(config); - } + Response->Record.MutableResponse()->mutable_identity()->set_cluster(Self->ClusterName); + Response->Record.MutableResponse()->mutable_identity()->set_version(Self->YamlVersion); + Response->Record.MutableResponse()->set_config(Self->YamlConfig); for (auto &[id, cfg] : Self->VolatileYamlConfigs) { auto *config = Response->Record.MutableResponse()->add_volatile_configs(); diff --git a/ydb/core/cms/console/console__get_yaml_metadata.cpp b/ydb/core/cms/console/console__get_yaml_metadata.cpp new file mode 100644 index 0000000000..0d183d8b42 --- /dev/null +++ b/ydb/core/cms/console/console__get_yaml_metadata.cpp @@ -0,0 +1,63 @@ +#include "console_configs_manager.h" + +#include <ydb/core/tablet_flat/tablet_flat_executed.h> + +namespace NKikimr::NConsole { + +using namespace NKikimrConsole; + +class TConfigsManager::TTxGetYamlMetadata : public TTransactionBase<TConfigsManager> { +public: + TTxGetYamlMetadata(TConfigsManager *self, + TEvConsole::TEvGetAllMetadataRequest::TPtr &ev) + : TBase(self) + , Request(std::move(ev)) + { + } + + bool Execute(TTransactionContext &, const TActorContext &) override + { + Response = MakeHolder<TEvConsole::TEvGetAllMetadataResponse>(); + + if (Self->YamlConfig) { + auto doc = NFyaml::TDocument::Parse(Self->YamlConfig); + + TStringStream metadata; + metadata << doc.Root().Map().at("metadata"); + + Response->Record.MutableResponse()->set_metadata(metadata.Str()); + + for (auto &[id, cfg] : Self->VolatileYamlConfigs) { + auto *config = Response->Record.MutableResponse()->add_volatile_configs(); + metadata.clear(); + auto doc = NFyaml::TDocument::Parse(cfg); + metadata << doc.Root().Map().at("metadata"); + config->set_id(id); + config->set_metadata(metadata.Str()); + } + + } + + return true; + } + + void Complete(const TActorContext &ctx) override + { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxGetYamlMetadata Complete"); + + ctx.Send(Request->Sender, Response.Release()); + + Self->TxProcessor->TxCompleted(this, ctx); + } + +private: + TEvConsole::TEvGetAllMetadataRequest::TPtr Request; + THolder<TEvConsole::TEvGetAllMetadataResponse> Response; +}; + +ITransaction *TConfigsManager::CreateTxGetYamlMetadata(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev) +{ + return new TTxGetYamlMetadata(this, ev); +} + +} // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp new file mode 100644 index 0000000000..8cf3a4ab13 --- /dev/null +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -0,0 +1,120 @@ +#include "console_configs_manager.h" +#include "console_configs_provider.h" + +#include <ydb/core/tablet_flat/tablet_flat_executed.h> + +namespace NKikimr::NConsole { + +using namespace NKikimrConsole; + +class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase<TConfigsManager> { +public: + TTxReplaceYamlConfig(TConfigsManager *self, + TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) + : TBase(self) + , Request(std::move(ev)) + { + } + + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override + { + auto &req = Request->Get()->Record; + + NIceDb::TNiceDb db(txc.DB); + + auto config = req.GetRequest().config(); + + try { + auto metadata = NYamlConfig::GetMetadata(config); + Cluster = metadata.Cluster.value_or(TString("unknown")); + Version = metadata.Version.value_or(0); + UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMetadata{ + .Version = Version + 1, + .Cluster = Cluster, + }); + + if (UpdatedConfig != Self->YamlConfig || Self->YamlDropped) { + Modify = true; + + auto tree = NFyaml::TDocument::Parse(UpdatedConfig); + auto resolved = NYamlConfig::ResolveAll(tree); + + if (Self->ClusterName != Cluster) { + ythrow yexception() << "ClusterName mismatch"; + } + + if (Version != Self->YamlVersion) { + ythrow yexception() << "Version mismatch"; + } + + for (auto& [_, config] : resolved.Configs) { + auto cfg = NYamlConfig::YamlToProto(config.second); + } + + if (!req.GetRequest().dry_run()) { + db.Table<Schema::YamlConfig>().Key(Version + 1) + .Update<Schema::YamlConfig::Config>(UpdatedConfig) + // set config dropped by default to support rollback to previous versions + // where new config layout is not supported + // it will lead to ignoring config from new versions + .Update<Schema::YamlConfig::Dropped>(true); + + /* Later we shift this boundary to support rollback and history */ + db.Table<Schema::YamlConfig>().Key(Version) + .Delete(); + } + } + + Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, new TEvConsole::TEvReplaceYamlConfigResponse()); + } catch (const yexception& ex) { + Error = true; + + auto ev = MakeHolder<TEvConsole::TEvGenericError>(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ex.what()); + Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, ev.Release()); + } + + return true; + } + + void Complete(const TActorContext &ctx) override + { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); + + auto &req = Request->Get()->Record; + + ctx.Send(Response.Release()); + + if (!Error && Modify && !req.GetRequest().dry_run()) { + Self->YamlVersion = Version + 1; + Self->YamlConfig = UpdatedConfig; + Self->YamlDropped = false; + + Self->VolatileYamlConfigs.clear(); + + auto resp = MakeHolder<TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig>(Self->YamlConfig); + ctx.Send(Self->ConfigsProvider, resp.Release()); + } + + Self->TxProcessor->TxCompleted(this, ctx); + } + +private: + TEvConsole::TEvReplaceYamlConfigRequest::TPtr Request; + THolder<NActors::IEventHandle> Response; + bool Error = false; + bool Modify = false; + ui32 Version; + TString Cluster; + TString UpdatedConfig; +}; + +ITransaction *TConfigsManager::CreateTxReplaceYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) +{ + return new TTxReplaceYamlConfig(this, ev); +} + +} // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index 78e56fee78..18208f6d8f 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -154,9 +154,10 @@ struct Schema : NIceDb::Schema { struct YamlConfig : Table<103> { struct Version : Column<1, NScheme::NTypeIds::Uint64> {}; struct Config : Column<2, NScheme::NTypeIds::String> {}; + struct Dropped : Column<3, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<Version>; - using TColumns = TableColumns<Version, Config>; + using TColumns = TableColumns<Version, Config, Dropped>; }; using TTables = SchemaTables<Config, Tenants, TenantPools, TenantUnits, RemovedTenants, diff --git a/ydb/core/cms/console/console__set_config.cpp b/ydb/core/cms/console/console__set_config.cpp index cd062e1cff..77604fa38d 100644 --- a/ydb/core/cms/console/console__set_config.cpp +++ b/ydb/core/cms/console/console__set_config.cpp @@ -1,9 +1,9 @@ -#include "config_index.h" #include "console_impl.h" #include "console_configs_manager.h" #include "console_tenants_manager.h" #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/util/config_index.h> namespace NKikimr::NConsole { diff --git a/ydb/core/cms/console/console__set_yaml_config.cpp b/ydb/core/cms/console/console__set_yaml_config.cpp new file mode 100644 index 0000000000..f219c8af99 --- /dev/null +++ b/ydb/core/cms/console/console__set_yaml_config.cpp @@ -0,0 +1,107 @@ +#include "console_configs_manager.h" +#include "console_configs_provider.h" + +#include <ydb/core/tablet_flat/tablet_flat_executed.h> + +namespace NKikimr::NConsole { + +using namespace NKikimrConsole; + +class TConfigsManager::TTxSetYamlConfig : public TTransactionBase<TConfigsManager> { +public: + TTxSetYamlConfig(TConfigsManager *self, + TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) + : TBase(self) + , Request(std::move(ev)) + { + } + + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override + { + auto &req = Request->Get()->Record; + + NIceDb::TNiceDb db(txc.DB); + + auto config = req.GetRequest().config(); + + try { + UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMetadata{ + .Version = Self->YamlVersion + 1, + .Cluster = Self->ClusterName, + }); + + if (UpdatedConfig != Self->YamlConfig || Self->YamlDropped) { + Modify = true; + + auto tree = NFyaml::TDocument::Parse(UpdatedConfig); + auto resolved = NYamlConfig::ResolveAll(tree); + + for (auto& [_, config] : resolved.Configs) { + auto cfg = NYamlConfig::YamlToProto(config.second); + } + + if (!req.GetRequest().dry_run()) { + db.Table<Schema::YamlConfig>().Key(Self->YamlVersion + 1) + .Update<Schema::YamlConfig::Config>(UpdatedConfig) + // set config dropped by default to support rollback to previous versions + // where new config layout is not supported + // it will lead to ignoring config from new versions + .Update<Schema::YamlConfig::Dropped>(true); + + /* Later we shift this boundary to support rollback and history */ + db.Table<Schema::YamlConfig>().Key(Self->YamlVersion) + .Delete(); + } + } + + Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, new TEvConsole::TEvSetYamlConfigResponse()); + } catch (const yexception& ex) { + Error = true; + + auto ev = MakeHolder<TEvConsole::TEvGenericError>(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ex.what()); + Response = MakeHolder<NActors::IEventHandle>(Request->Sender, ctx.SelfID, ev.Release()); + } + + return true; + } + + void Complete(const TActorContext &ctx) override + { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxSetYamlConfig Complete"); + + auto &req = Request->Get()->Record; + + ctx.Send(Response.Release()); + + if (!Error && Modify && !req.GetRequest().dry_run()) { + Self->YamlVersion = Self->YamlVersion + 1; + Self->YamlConfig = UpdatedConfig; + Self->YamlDropped = false; + + Self->VolatileYamlConfigs.clear(); + + auto resp = MakeHolder<TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig>(Self->YamlConfig); + ctx.Send(Self->ConfigsProvider, resp.Release()); + } + + Self->TxProcessor->TxCompleted(this, ctx); + } + +private: + TEvConsole::TEvSetYamlConfigRequest::TPtr Request; + THolder<NActors::IEventHandle> Response; + bool Error = false; + bool Modify = false; + TString UpdatedConfig; +}; + +ITransaction *TConfigsManager::CreateTxSetYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) +{ + return new TTxSetYamlConfig(this, ev); +} + +} // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console__toggle_config_validator.cpp b/ydb/core/cms/console/console__toggle_config_validator.cpp index fb2dfb20e7..8731b6806c 100644 --- a/ydb/core/cms/console/console__toggle_config_validator.cpp +++ b/ydb/core/cms/console/console__toggle_config_validator.cpp @@ -1,9 +1,9 @@ -#include "config_index.h" #include "console_impl.h" #include "console_configs_manager.h" #include "console_tenants_manager.h" #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/cms/console/validators/registry.h> namespace NKikimr::NConsole { diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 177dda3dd5..b445848dd6 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -1,4 +1,6 @@ #include "console_configs_manager.h" + +#include "configs_dispatcher.h" #include "console_configs_provider.h" #include "console_impl.h" #include "http.h" @@ -360,9 +362,11 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc, } if (!yamlConfigRowset.EndOfSet()) { - auto config = yamlConfigRowset.template GetValue<Schema::YamlConfig::Config>(); YamlVersion = yamlConfigRowset.template GetValue<Schema::YamlConfig::Version>(); - YamlConfig = config; + YamlConfig = yamlConfigRowset.template GetValue<Schema::YamlConfig::Config>(); + // ignore this as deprecated + // now used only for disabling new config layout for older console + YamlDropped = false; } while (!configItemRowset.EndOfSet()) { @@ -614,9 +618,19 @@ void TConfigsManager::Handle(TEvConsole::TEvToggleConfigValidatorRequest::TPtr & TxProcessor->ProcessTx(CreateTxToggleConfigValidator(ev), ctx); } -void TConfigsManager::Handle(TEvConsole::TEvApplyConfigRequest::TPtr &ev, const TActorContext &ctx) +void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, const TActorContext &ctx) +{ + TxProcessor->ProcessTx(CreateTxReplaceYamlConfig(ev), ctx); +} + +void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, const TActorContext &ctx) { - TxProcessor->ProcessTx(CreateTxApplyYamlConfig(ev), ctx); + TxProcessor->ProcessTx(CreateTxSetYamlConfig(ev), ctx); +} + +void TConfigsManager::Handle(TEvConsole::TEvDropConfigRequest::TPtr &ev, const TActorContext &ctx) +{ + TxProcessor->ProcessTx(CreateTxDropYamlConfig(ev), ctx); } void TConfigsManager::Handle(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev, const TActorContext &ctx) @@ -624,23 +638,37 @@ void TConfigsManager::Handle(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev, cons TxProcessor->ProcessTx(CreateTxGetYamlConfig(ev), ctx); } +void TConfigsManager::Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev, const TActorContext &ctx) +{ + if (!AppData()->FeatureFlags.GetEnableGetNodeLabels()) { + auto response = MakeHolder<TEvConsole::TEvDisabled>(); + ctx.Send(ev->Sender, response.Release()); + } else { + ctx.Send(ev->Forward(MakeConfigsDispatcherID(ev->Get()->Record.GetRequest().node_id()))); + } +} + +void TConfigsManager::Handle(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev, const TActorContext &ctx) +{ + TxProcessor->ProcessTx(CreateTxGetYamlMetadata(ev), ctx); +} + void TConfigsManager::Handle(TEvConsole::TEvResolveConfigRequest::TPtr &ev, const TActorContext &ctx) { auto &rec = ev->Get()->Record.GetRequest(); try { auto config = rec.config(); - auto parser = NFyaml::TParser::Create(config); - parser.NextDocument(); - auto tree = parser.NextDocument(); - - if (!tree) { - ythrow yexception() << "Empty YAML config"; - } + auto tree = NFyaml::TDocument::Parse(config); for (auto &volatileConfig : rec.volatile_configs()) { auto str = volatileConfig.config(); auto d = NFyaml::TDocument::Parse(str); - NYamlConfig::AppendVolatileConfigs(tree.value(), d); + if (d.Root().Type() != NFyaml::ENodeType::Sequence) { + auto node = d.Root().Map().at("selector_config"); + NYamlConfig::AppendVolatileConfigs(tree, node); + } else { + NYamlConfig::AppendVolatileConfigs(tree, d); + } } TSet<NYamlConfig::TNamedLabel> namedLabels; @@ -648,28 +676,23 @@ void TConfigsManager::Handle(TEvConsole::TEvResolveConfigRequest::TPtr &ev, cons namedLabels.insert(NYamlConfig::TNamedLabel{label.label(), label.value()}); } - auto resolved = NYamlConfig::Resolve(tree.value(), namedLabels); + auto resolved = NYamlConfig::Resolve(tree, namedLabels); - auto Response = MakeHolder<TEvConsole::TEvResolveConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); + auto response = MakeHolder<TEvConsole::TEvResolveConfigResponse>(); TStringStream resolvedStr; resolvedStr << resolved.second; - Response->Record.MutableResponse()->set_config(resolvedStr.Str()); + response->Record.MutableResponse()->set_config(resolvedStr.Str()); - ctx.Send(ev->Sender, Response.Release()); + ctx.Send(ev->Sender, response.Release()); } catch (const yexception& ex) { - auto Response = MakeHolder<TEvConsole::TEvResolveConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::BAD_REQUEST); - op->set_ready(true); - auto *issue = op->add_issues(); + auto response = MakeHolder<TEvConsole::TEvGenericError>(); + response->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = response->Record.AddIssues(); issue->set_severity(NYql::TSeverityIds::S_ERROR); issue->set_message(ex.what()); - ctx.Send(ev->Sender, Response.Release()); + ctx.Send(ev->Sender, response.Release()); } } @@ -678,162 +701,190 @@ void TConfigsManager::Handle(TEvConsole::TEvResolveAllConfigRequest::TPtr &ev, c auto &rec = ev->Get()->Record.GetRequest(); try { auto config = rec.config(); - auto parser = NFyaml::TParser::Create(config); - parser.NextDocument(); - auto tree = parser.NextDocument(); - - if (!tree) { - ythrow yexception() << "Empty YAML config"; - } + auto tree = NFyaml::TDocument::Parse(config); for (auto &volatileConfig : rec.volatile_configs()) { auto str = volatileConfig.config(); auto d = NFyaml::TDocument::Parse(str); - NYamlConfig::AppendVolatileConfigs(tree.value(), d); + if (d.Root().Type() != NFyaml::ENodeType::Sequence) { + auto node = d.Root().Map().at("selector_config"); + NYamlConfig::AppendVolatileConfigs(tree, node); + } else { + NYamlConfig::AppendVolatileConfigs(tree, d); + } } - auto resolved = NYamlConfig::ResolveAll(tree.value()); + auto resolved = NYamlConfig::ResolveAll(tree); auto Response = MakeHolder<TEvConsole::TEvResolveAllConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); - TStringStream resolvedStr; + auto convert = [] (const NYamlConfig::TLabel::EType& label) -> Ydb::DynamicConfig::YamlLabelExt::LabelType { + switch(label) { + case NYamlConfig::TLabel::EType::Negative: + return Ydb::DynamicConfig::YamlLabelExt::NOT_SET; + case NYamlConfig::TLabel::EType::Common: + return Ydb::DynamicConfig::YamlLabelExt::COMMON; + case NYamlConfig::TLabel::EType::Empty: + return Ydb::DynamicConfig::YamlLabelExt::EMPTY; + default: + Y_FAIL("unexpected enum value"); + } + }; + + if (!rec.verbose_response()) { + TStringStream resolvedStr; - bool first = true; + bool first = true; - for (auto &[labelSets, config] : resolved.Configs) { - if (!first) { - resolvedStr << "\n"; + for (auto &[labelSets, config] : resolved.Configs) { + if (!first) { + resolvedStr << "\n"; + } + resolvedStr << "---\n"; + resolvedStr << "# applicable to: \n"; + for (auto &labelSet : labelSets) { + resolvedStr << "# ["; + for (size_t i = 0; i < labelSet.size(); ++i) { + auto &label = labelSet[i]; + resolvedStr << resolved.Labels[i] << ":" << ( + label.Type == NYamlConfig::TLabel::EType::Common ? label.Value.Quote() : + label.Type == NYamlConfig::TLabel::EType::Negative ? " - " : + " _ " ) + << ", "; + } + resolvedStr << "] \n"; + } + resolvedStr << config.second << Endl; + first = false; } - resolvedStr << "---\n"; - resolvedStr << "# applicable to: \n"; - for (auto &labelSet : labelSets) { - resolvedStr << "# ["; - for (size_t i = 0; i < labelSet.size(); ++i) { - auto &label = labelSet[i]; - resolvedStr << resolved.Labels[i] << ":" << ( - label.Type == NYamlConfig::TLabel::EType::Common ? label.Value.Quote() : - label.Type == NYamlConfig::TLabel::EType::Negative ? " - " : - " _ " ) - << ", "; + + Response->Record.MutableResponse()->set_config(resolvedStr.Str()); + } else { + for (auto &[labelSets, config] : resolved.Configs) { + auto *serConfig = Response->Record.MutableResponse()->add_configs(); + for (auto &labelSet : labelSets) { + auto *serLabelSet = serConfig->add_label_sets(); + for (size_t i = 0; i < labelSet.size(); ++i) { + auto &label = labelSet[i]; + auto &name = resolved.Labels[i]; + auto* serLabel = serLabelSet->add_labels(); + serLabel->set_label(name); + serLabel->set_type(convert(label.Type)); + serLabel->set_value(label.Value); + } } - resolvedStr << "] \n"; + TStringStream configStr; + configStr << config.second; + serConfig->set_config(configStr.Str()); } - resolvedStr << config.second << Endl; - first = false; } - Response->Record.MutableResponse()->set_config(resolvedStr.Str()); - ctx.Send(ev->Sender, Response.Release()); } catch (const yexception& ex) { - auto Response = MakeHolder<TEvConsole::TEvResolveAllConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::BAD_REQUEST); - op->set_ready(true); - auto *issue = op->add_issues(); + auto response = MakeHolder<TEvConsole::TEvGenericError>(); + response->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = response->Record.AddIssues(); issue->set_severity(NYql::TSeverityIds::S_ERROR); issue->set_message(ex.what()); - ctx.Send(ev->Sender, Response.Release()); + ctx.Send(ev->Sender, response.Release()); } } void TConfigsManager::Handle(TEvConsole::TEvAddVolatileConfigRequest::TPtr &ev, const TActorContext &ctx) { auto &rec = ev->Get()->Record.GetRequest(); - auto Response = MakeHolder<TEvConsole::TEvAddVolatileConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); - if (VolatileYamlConfigs.empty() || VolatileYamlConfigs.rbegin()->first + 1 == rec.id()) { - try { - auto cfg = rec.config(); - auto doc = NFyaml::TDocument::Parse(cfg); - NYamlConfig::ValidateVolatileConfig(doc); - auto config = YamlConfig; - auto parser = NFyaml::TParser::Create(config); - parser.NextDocument(); - auto tree = parser.NextDocument(); + try { + auto response = MakeHolder<TEvConsole::TEvAddVolatileConfigResponse>(); + auto cfg = rec.config(); + auto metadata = NYamlConfig::GetVolatileMetadata(cfg); - if (!tree) { - ythrow yexception() << "Empty YAML config"; - } + if (!metadata.Id || !metadata.Cluster || !metadata.Version) { + ythrow yexception() << "Invalid metadata"; + } + + cfg = NYamlConfig::ReplaceMetadata(cfg, metadata); + + auto clusterName = metadata.Cluster.value(); + auto id = metadata.Id.value(); + auto version = metadata.Version.value(); + + auto doc = NFyaml::TDocument::Parse(cfg); + NYamlConfig::ValidateVolatileConfig(doc); + auto node = doc.Root().Map().at("selector_config"); + + if (VolatileYamlConfigs.empty() || VolatileYamlConfigs.rbegin()->first + 1 == id) { + auto config = YamlConfig; + auto tree = NFyaml::TDocument::Parse(config); for (auto &[_, config] : VolatileYamlConfigs) { - auto d = NFyaml::TDocument::Parse(config); - NYamlConfig::AppendVolatileConfigs(tree.value(), d); + auto doc = NFyaml::TDocument::Parse(config); + auto node = doc.Root().Map().at("selector_config"); + NYamlConfig::AppendVolatileConfigs(tree, node); } - NYamlConfig::AppendVolatileConfigs(tree.value(), doc); + NYamlConfig::AppendVolatileConfigs(tree, node); - auto resolved = NYamlConfig::ResolveAll(tree.value()); + auto resolved = NYamlConfig::ResolveAll(tree); for (auto &[_, config] : resolved.Configs) { auto cfg = NYamlConfig::YamlToProto(config.second); } - if (ClusterName != rec.cluster()) { + if (ClusterName != clusterName) { ythrow yexception() << "ClusterName mismatch"; } - if (YamlVersion != rec.version()) { + if (YamlVersion != version) { ythrow yexception() << "Version mismatch"; } - VolatileYamlConfigs.try_emplace(rec.id(), rec.config()); - - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); + VolatileYamlConfigs.try_emplace(id, cfg); auto resp = MakeHolder<TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig>( YamlConfig, VolatileYamlConfigs); ctx.Send(ConfigsProvider, resp.Release()); - } catch (const yexception& ex) { - op->set_status(Ydb::StatusIds::BAD_REQUEST); - op->set_ready(true); - auto *issue = op->add_issues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(ex.what()); + } else if (auto it = VolatileYamlConfigs.find(id); it == VolatileYamlConfigs.end() || it->second != cfg) { + ythrow yexception() << "Config already exists"; } - } else if (auto it = VolatileYamlConfigs.find(rec.id()); it != VolatileYamlConfigs.end() && it->second == rec.config()) { - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); - } else { - op->set_status(Ydb::StatusIds::BAD_REQUEST); - op->set_ready(true); - } - ctx.Send(ev->Sender, Response.Release()); + ctx.Send(ev->Sender, response.Release()); + } catch (const yexception& ex) { + auto response = MakeHolder<TEvConsole::TEvGenericError>(); + response->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = response->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ex.what()); + ctx.Send(ev->Sender, response.Release()); + } } void TConfigsManager::Handle(TEvConsole::TEvRemoveVolatileConfigRequest::TPtr &ev, const TActorContext &ctx) { auto &rec = ev->Get()->Record.GetRequest(); - auto Response = MakeHolder<TEvConsole::TEvRemoveVolatileConfigResponse>(); - auto *op = Response->Record.MutableResponse()->mutable_operation(); try { - if (ClusterName != rec.cluster()) { - ythrow yexception() << "ClusterName mismatch"; - } + if (!rec.force()) { + if (ClusterName != rec.identity().cluster()) { + ythrow yexception() << "ClusterName mismatch"; + } - if (YamlVersion != rec.version()) { - ythrow yexception() << "Version mismatch"; + if (YamlVersion != rec.identity().version()) { + ythrow yexception() << "Version mismatch"; + } } - int toRemove = 0; - for (auto &id : rec.ids()) { + for (auto &id : rec.ids().ids()) { toRemove += (VolatileYamlConfigs.find(id) != VolatileYamlConfigs.end()) ? 1 : 0; } - if (!rec.ids_size()) { + if (rec.all()) { VolatileYamlConfigs.clear(); - } else if (rec.ids_size() == toRemove) { - for (auto &id : rec.ids()) { + } else if (rec.ids().ids_size() == toRemove) { + for (auto &id : rec.ids().ids()) { VolatileYamlConfigs.erase(id); } } else { @@ -845,17 +896,16 @@ void TConfigsManager::Handle(TEvConsole::TEvRemoveVolatileConfigRequest::TPtr &e VolatileYamlConfigs); ctx.Send(ConfigsProvider, resp.Release()); - op->set_status(Ydb::StatusIds::SUCCESS); - op->set_ready(true); + auto response = MakeHolder<TEvConsole::TEvRemoveVolatileConfigResponse>(); + ctx.Send(ev->Sender, response.Release()); } catch (const yexception& ex) { - op->set_status(Ydb::StatusIds::BAD_REQUEST); - op->set_ready(true); - auto *issue = op->add_issues(); + auto response = MakeHolder<TEvConsole::TEvGenericError>(); + response->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = response->Record.AddIssues(); issue->set_severity(NYql::TSeverityIds::S_ERROR); issue->set_message(ex.what()); + ctx.Send(ev->Sender, response.Release()); } - - ctx.Send(ev->Sender, Response.Release()); } void TConfigsManager::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev, const TActorContext &ctx) diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index e9f01d3b66..81cb5c3e2c 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -2,7 +2,6 @@ #include "defs.h" -#include "config_index.h" #include "configs_config.h" #include "console.h" #include "logger.h" @@ -11,6 +10,7 @@ #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <library/cpp/actors/core/hfunc.h> @@ -109,8 +109,11 @@ private: class TTxUpdateLastProvidedConfig; class TTxGetLogTail; class TTxLogCleanup; - class TTxApplyYamlConfig; + class TTxReplaceYamlConfig; + class TTxSetYamlConfig; + class TTxDropYamlConfig; class TTxGetYamlConfig; + class TTxGetYamlMetadata; ITransaction *CreateTxAddConfigSubscription(TEvConsole::TEvAddConfigSubscriptionRequest::TPtr &ev); ITransaction *CreateTxCleanupSubscriptions(TEvInterconnect::TEvNodesInfo::TPtr &ev); @@ -122,8 +125,11 @@ private: ITransaction *CreateTxUpdateLastProvidedConfig(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); ITransaction *CreateTxGetLogTail(TEvConsole::TEvGetLogTailRequest::TPtr &ev); ITransaction *CreateTxLogCleanup(); - ITransaction *CreateTxApplyYamlConfig(TEvConsole::TEvApplyConfigRequest::TPtr &ev); + ITransaction *CreateTxReplaceYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxSetYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxDropYamlConfig(TEvConsole::TEvDropConfigRequest::TPtr &ev); ITransaction *CreateTxGetYamlConfig(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev); + ITransaction *CreateTxGetYamlMetadata(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev); void Handle(TEvConsole::TEvAddConfigSubscriptionRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvConfigNotificationResponse::TPtr &ev, const TActorContext &ctx); @@ -134,13 +140,17 @@ private: void Handle(TEvConsole::TEvReplaceConfigSubscriptionsRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvToggleConfigValidatorRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvGetLogTailRequest::TPtr &ev, const TActorContext &ctx); + void Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvResolveConfigRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvResolveAllConfigRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev, const TActorContext &ctx); + void Handle(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvAddVolatileConfigRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvRemoveVolatileConfigRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev, const TActorContext &ctx); - void Handle(TEvConsole::TEvApplyConfigRequest::TPtr & ev, const TActorContext & ctx); + void Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr & ev, const TActorContext & ctx); + void Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr & ev, const TActorContext & ctx); + void Handle(TEvConsole::TEvDropConfigRequest::TPtr & ev, const TActorContext & ctx); void Handle(TEvPrivate::TEvStateLoaded::TPtr &ev, const TActorContext &ctx); void Handle(TEvPrivate::TEvCleanupSubscriptions::TPtr &ev, const TActorContext &ctx); @@ -151,13 +161,7 @@ private: if (CheckRights(ev->Get()->Record.GetUserToken())) { Handle(ev, ctx); } else { - auto req = MakeHolder<typename std::decay_t<decltype(*ev->Get())>::TResponse>(); - auto *op = req->Record.MutableResponse()->mutable_operation(); - op->set_status(Ydb::StatusIds::UNAUTHORIZED); - op->set_ready(true); - auto issue = op->add_issues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message("User must have administrator rights"); + auto req = MakeHolder<TEvConsole::TEvUnauthorized>(); ctx.Send(ev->Sender, req.Release()); } } @@ -180,10 +184,14 @@ private: HFunc(TEvConsole::TEvResolveConfigRequest, Handle); HFunc(TEvConsole::TEvResolveAllConfigRequest, Handle); HFunc(TEvConsole::TEvGetAllConfigsRequest, HandleWithRights); + HFunc(TEvConsole::TEvGetNodeLabelsRequest, HandleWithRights); + HFunc(TEvConsole::TEvGetAllMetadataRequest, HandleWithRights); HFunc(TEvConsole::TEvAddVolatileConfigRequest, HandleWithRights); HFunc(TEvConsole::TEvRemoveVolatileConfigRequest, HandleWithRights); FFunc(TEvConsole::EvGetConfigItemsRequest, ForwardToConfigsProvider); - HFuncTraced(TEvConsole::TEvApplyConfigRequest, HandleWithRights); + HFuncTraced(TEvConsole::TEvReplaceYamlConfigRequest, HandleWithRights); + HFuncTraced(TEvConsole::TEvSetYamlConfigRequest, HandleWithRights); + HFuncTraced(TEvConsole::TEvDropConfigRequest, HandleWithRights); FFunc(TEvConsole::EvGetConfigSubscriptionRequest, ForwardToConfigsProvider); FFunc(TEvConsole::EvGetNodeConfigItemsRequest, ForwardToConfigsProvider); FFunc(TEvConsole::EvGetNodeConfigRequest, ForwardToConfigsProvider); @@ -249,6 +257,7 @@ private: TString ClusterName; ui32 YamlVersion = 0; TString YamlConfig; + bool YamlDropped = false; TMap<ui64, TString> VolatileYamlConfigs; }; diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index bc675b6fcf..e346e0a96a 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -747,6 +747,7 @@ void TConfigsProvider::DumpStateHTML(IOutputStream &os) const { << " ItemKinds: " << KindsToString(s->ItemKinds) << Endl << " ConfigVersions: " << s->LastProvided.ShortDebugString() << Endl << " ServeYaml: " << s->ServeYaml << Endl + << " YamlApiVersion: " << s->YamlApiVersion << Endl << " YamlVersion: " << s->YamlConfigVersion << ".["; bool first = true; for (auto &[id, hash] : s->VolatileYamlConfigHashes) { @@ -804,8 +805,9 @@ void TConfigsProvider::Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev subscription->ItemKinds.insert(rec.GetConfigItemKinds().begin(), rec.GetConfigItemKinds().end()); subscription->LastProvided.CopyFrom(rec.GetKnownVersion()); - if (rec.HasServeYaml()) { + if (rec.HasServeYaml() && rec.GetServeYaml() && rec.HasYamlApiVersion() && rec.GetYamlApiVersion() == 1) { subscription->ServeYaml = rec.GetServeYaml(); + subscription->YamlApiVersion = rec.GetYamlApiVersion(); subscription->YamlConfigVersion = rec.GetYamlVersion(); for (auto &volatileConfigVersion : rec.GetVolatileYamlVersion()) { subscription->VolatileYamlConfigHashes[volatileConfigVersion.GetId()] = volatileConfigVersion.GetHash(); @@ -1088,7 +1090,7 @@ void TConfigsProvider::Handle(TEvConsole::TEvGetNodeConfigRequest::TPtr &ev, con LOG_TRACE_S(ctx, NKikimrServices::CMS_CONFIGS, "Send TEvGetNodeConfigResponse: " << response->Record.ShortDebugString()); - if (rec.HasServeYaml() && rec.GetServeYaml()) { + if (rec.HasServeYaml() && rec.GetServeYaml() && rec.HasYamlApiVersion() && rec.GetYamlApiVersion() == 1) { response->Record.SetYamlConfig(YamlConfig); for (auto &[id, config] : VolatileYamlConfigs) { @@ -1221,12 +1223,17 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateSubscriptions::TPtr &ev, cons void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const TActorContext &ctx) { YamlConfig = ev->Get()->YamlConfig; - VolatileYamlConfigs = ev->Get()->VolatileYamlConfigs; + VolatileYamlConfigs.clear(); YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); VolatileYamlConfigHashes.clear(); - for (auto& [id, config] : VolatileYamlConfigs) { - VolatileYamlConfigHashes[id] = THash<TString>()(config); + for (auto& [id, config] : ev->Get()->VolatileYamlConfigs) { + auto doc = NFyaml::TDocument::Parse(config); + // we strip it to provide old format for config dispatcher + auto node = doc.Root().Map().at("selector_config"); + TString strippedConfig = "\n" + config.substr(node.BeginMark().InputPos, node.EndMark().InputPos - node.BeginMark().InputPos) + "\n"; + VolatileYamlConfigs[id] = strippedConfig; + VolatileYamlConfigHashes[id] = THash<TString>()(strippedConfig); } for (auto &[_, subscription] : InMemoryIndex.GetSubscriptions()) { diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index c8e270d2d3..5b6ed9ead7 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -3,10 +3,10 @@ #include "defs.h" #include "configs_config.h" -#include "config_index.h" #include "console.h" #include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <library/cpp/actors/core/hfunc.h> diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index f3e9725ae1..171f9d2722 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -1,10 +1,10 @@ #include "console_configs_subscriber.h" #include "console.h" -#include "config_index.h" #include "util.h" #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/mind/tenant_pool.h> #include <library/cpp/actors/core/actor_bootstrapped.h> @@ -45,6 +45,7 @@ public: const TVector<ui32> &kinds, const NKikimrConfig::TAppConfig ¤tConfig, bool processYaml, + ui64 version, const TString &yamlConfig, const TMap<ui64, TString> &volatileYamlConfigs) : OwnerId(ownerId) @@ -55,6 +56,7 @@ public: , LastOrder(0) , CurrentConfig(currentConfig) , ServeYaml(processYaml) + , Version(version) , YamlConfig(yamlConfig) , VolatileYamlConfigs(volatileYamlConfigs) { @@ -93,6 +95,7 @@ public: switch (ev->GetTypeRewrite()) { HFuncTraced(TEvPrivate::TEvRetryPoolStatus, Handle); HFuncTraced(TEvTenantPool::TEvTenantPoolStatus, Handle); + HFuncTraced(TEvConsole::TEvGetNodeConfigResponse, Handle); HFuncTraced(TEvConsole::TEvConfigSubscriptionResponse, Handle); HFuncTraced(TEvConsole::TEvConfigSubscriptionError, Handle); HFuncTraced(TEvConsole::TEvConfigSubscriptionNotification, Handle); @@ -149,6 +152,32 @@ public: Generation = 0; Die(ctx); } + + if (!FirstUpdateSent) { + auto request = MakeHolder<TEvConsole::TEvGetNodeConfigRequest>(); + request->Record.MutableNode()->SetNodeId(SelfId().NodeId()); + request->Record.MutableNode()->SetHost(FQDNHostName()); + request->Record.MutableNode()->SetTenant(Tenant); + request->Record.MutableNode()->SetNodeType(NodeType); + for (auto &kind : Kinds) { + request->Record.AddItemKinds(kind); + } + + NTabletPipe::SendData(ctx, Pipe, request.Release(), Cookie); + } + } + + void Handle(TEvConsole::TEvGetNodeConfigResponse::TPtr &ev, const TActorContext &ctx) { + if (!FirstUpdateSent) { + ctx.ExecutorThread.Send( + new NActors::IEventHandle( + SelfId(), + ev->Sender, + new NConsole::TEvConsole::TEvConfigSubscriptionNotification( + Generation, + ev->Get()->Record.GetConfig(), + THashSet<ui32>(Kinds.begin(), Kinds.end())))); + } } void Handle(TEvConsole::TEvConfigSubscriptionError::TPtr &ev, const TActorContext &ctx) { @@ -185,6 +214,7 @@ public: YamlConfig = rec.GetYamlConfig(); YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); } + notChanged = false; } @@ -309,6 +339,7 @@ private: request->Record.MutableOptions()->SetNodeType(NodeType); request->Record.MutableOptions()->SetHost(FQDNHostName()); request->Record.SetServeYaml(ServeYaml); + request->Record.SetYamlApiVersion(Version); for (auto &kind : Kinds) request->Record.AddConfigItemKinds(kind); @@ -358,6 +389,7 @@ private: NKikimrConfig::TAppConfig CurrentConfig; bool ServeYaml = false; + ui64 Version; TString YamlConfig; TMap<ui64, TString> VolatileYamlConfigs; ui64 YamlConfigVersion = 0; @@ -378,10 +410,11 @@ IActor *CreateConfigsSubscriber( const NKikimrConfig::TAppConfig ¤tConfig, ui64 cookie, bool processYaml, + ui64 version, const TString &yamlConfig, const TMap<ui64, TString> &volatileYamlConfigs) { - return new TConfigsSubscriber(ownerId, cookie, kinds, currentConfig, processYaml, yamlConfig, volatileYamlConfigs); + return new TConfigsSubscriber(ownerId, cookie, kinds, currentConfig, processYaml, version, yamlConfig, volatileYamlConfigs); } } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console_configs_subscriber.h b/ydb/core/cms/console/console_configs_subscriber.h index d10abea137..52d0feb81c 100644 --- a/ydb/core/cms/console/console_configs_subscriber.h +++ b/ydb/core/cms/console/console_configs_subscriber.h @@ -14,6 +14,7 @@ IActor *CreateConfigsSubscriber( const NKikimrConfig::TAppConfig ¤tConfig, ui64 cookie = 0, bool processYaml = false, + ui64 version = 0, const TString &yamlConfig = {}, const TMap<ui64, TString> &volatileYamlConfigs = {}); diff --git a/ydb/core/cms/console/console_impl.h b/ydb/core/cms/console/console_impl.h index 648ed3801b..385ced8107 100644 --- a/ydb/core/cms/console/console_impl.h +++ b/ydb/core/cms/console/console_impl.h @@ -92,6 +92,7 @@ private: FFunc(TEvConsole::EvConfigNotificationResponse, ForwardToConfigsManager); FFunc(TEvConsole::EvConfigureRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvGetAllConfigsRequest, ForwardToConfigsManager); + FFunc(TEvConsole::EvGetAllMetadataRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvAddVolatileConfigRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvRemoveVolatileConfigRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvGetLogTailRequest, ForwardToConfigsManager); @@ -99,7 +100,10 @@ private: FFunc(TEvConsole::EvDescribeTenantOptionsRequest, ForwardToTenantsManager); FFunc(TEvConsole::EvGetConfigItemsRequest, ForwardToConfigsManager); HFuncTraced(TEvConsole::TEvGetConfigRequest, Handle); - FFunc(TEvConsole::EvApplyConfigRequest, ForwardToConfigsManager); + FFunc(TEvConsole::EvReplaceYamlConfigRequest, ForwardToConfigsManager); + FFunc(TEvConsole::EvGetNodeLabelsRequest, ForwardToConfigsManager); + FFunc(TEvConsole::EvSetYamlConfigRequest, ForwardToConfigsManager); + FFunc(TEvConsole::EvDropConfigRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvResolveConfigRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvResolveAllConfigRequest, ForwardToConfigsManager); FFunc(TEvConsole::EvGetConfigSubscriptionRequest, ForwardToConfigsManager); diff --git a/ydb/core/cms/console/console_ut_configs.cpp b/ydb/core/cms/console/console_ut_configs.cpp index b877b65dfc..47496042d3 100644 --- a/ydb/core/cms/console/console_ut_configs.cpp +++ b/ydb/core/cms/console/console_ut_configs.cpp @@ -654,9 +654,27 @@ NKikimrConsole::TConfigItem ITEM_DOMAIN_TENANT_POOL_2; const TString YAML_CONFIG_1 = R"( --- -cluster: "" -version: 1 +metadata: + cluster: "" + version: 0 +config: + log_config: + cluster_name: cluster1 +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + +const TString YAML_CONFIG_1_UPDATED = R"( --- +metadata: + kind: MainConfig + cluster: "" + version: 1 config: log_config: cluster_name: cluster1 @@ -671,9 +689,27 @@ selector_config: [] const TString YAML_CONFIG_2 = R"( --- -cluster: "" -version: 2 +metadata: + cluster: "" + version: 1 +config: + log_config: + cluster_name: cluster2 +allowed_labels: + test: + type: enum + values: + ? true + +selector_config: [] +)"; + +const TString YAML_CONFIG_2_UPDATED = R"( --- +metadata: + kind: MainConfig + cluster: "" + version: 2 config: log_config: cluster_name: cluster2 @@ -697,6 +733,23 @@ const TString VOLATILE_YAML_CONFIG_1_1 = R"( enable: false )"; +const TString EXTENDED_VOLATILE_YAML_CONFIG_1_1 = R"(metadata: + kind: VolatileConfig + cluster: "" + version: 1 + id: 0 +selector_config: + +- description: test 4 + selector: + tenant: /slice + config: + yaml_config_enabled: true + cms_config: + sentinel_config: + enable: false +)"; + const size_t VOLATILE_YAML_CONFIG_1_1_HASH = THash<TString>{}(VOLATILE_YAML_CONFIG_1_1); const TString VOLATILE_YAML_CONFIG_1_2 = R"( @@ -3848,7 +3901,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster-2"); @@ -3883,7 +3936,8 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { TVector<ui32>({(ui32)NKikimrConsole::TConfigItem::LogConfigItem}), NKikimrConfig::TAppConfig(), 0, - true); + true, + 1); runtime.Register(subscriber, 1); TDispatchOptions options1; @@ -3903,23 +3957,32 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); notification = runtime.GrabEdgeEventRethrow<TEvConsole::TEvConfigSubscriptionNotification>(edgeId); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1_UPDATED); CheckAddVolatileConfig(runtime, Ydb::StatusIds::SUCCESS, "", 1, 0, VOLATILE_YAML_CONFIG_1_1); notification = runtime.GrabEdgeEventRethrow<TEvConsole::TEvConfigSubscriptionNotification>(edgeId); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1_UPDATED); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.VolatileConfigsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[0].GetConfig(), VOLATILE_YAML_CONFIG_1_1); - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); + runtime.SendToConsole(new TEvConsole::TEvGetAllConfigsRequest()); + TAutoPtr<IEventHandle> handle; + auto configs = runtime.GrabEdgeEventRethrow<TEvConsole::TEvGetAllConfigsResponse>(handle); + UNIT_ASSERT_VALUES_EQUAL(configs->Record.GetResponse().identity().cluster(), ""); + UNIT_ASSERT_VALUES_EQUAL(configs->Record.GetResponse().identity().version(), 1); + UNIT_ASSERT_VALUES_EQUAL(configs->Record.GetResponse().config(), YAML_CONFIG_1_UPDATED); + UNIT_ASSERT_VALUES_EQUAL(configs->Record.GetResponse().volatile_configs_size(), 1); + UNIT_ASSERT_VALUES_EQUAL(configs->Record.GetResponse().volatile_configs(0).config(), EXTENDED_VOLATILE_YAML_CONFIG_1_1); + + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); CheckAddVolatileConfig(runtime, Ydb::StatusIds::SUCCESS, "", 1, 0, VOLATILE_YAML_CONFIG_1_1); ITEM_DOMAIN_LOG_2.MutableConfig()->MutableLogConfig()->SetClusterName("cluster-2"); @@ -3937,7 +4000,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1_UPDATED); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.VolatileConfigsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[0].GetConfig(), VOLATILE_YAML_CONFIG_1_1); @@ -3946,7 +4009,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1_UPDATED); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.VolatileConfigsSize(), 2); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[0].GetConfig(), VOLATILE_YAML_CONFIG_1_1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[1].GetConfig(), VOLATILE_YAML_CONFIG_1_2); @@ -3956,16 +4019,16 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1_UPDATED); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.VolatileConfigsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[0].GetConfig(), VOLATILE_YAML_CONFIG_1_2); - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_2); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_2); notification = runtime.GrabEdgeEventRethrow<TEvConsole::TEvConfigSubscriptionNotification>(edgeId); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), 1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_2); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_2_UPDATED); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.VolatileConfigsSize(), 0); } @@ -3984,12 +4047,13 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { event->Record.MutableOptions()->SetTenant("tenant-1"); event->Record.MutableOptions()->SetNodeType("type1"); event->Record.SetServeYaml(true); + event->Record.SetYamlApiVersion(1); event->Record.AddConfigItemKinds(NKikimrConsole::TConfigItem::LogConfigItem); runtime.SendToPipe(MakeConsoleID(0), edgeId, event, 0, GetPipeConfigWithRetries()); runtime.GrabEdgeEventRethrow<TEvConsole::TEvConfigSubscriptionNotification>(edgeId); // initial update - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); auto notification = runtime.GrabEdgeEventRethrow<TEvConsole::TEvConfigSubscriptionNotification>(edgeId); CheckAddVolatileConfig(runtime, Ydb::StatusIds::SUCCESS, "", 1, 0, VOLATILE_YAML_CONFIG_1_1); @@ -4032,7 +4096,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { ui32 generation = 1; TActorId edgeId = runtime.Sender; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); CheckAddVolatileConfig(runtime, Ydb::StatusIds::SUCCESS, "", 1, 0, VOLATILE_YAML_CONFIG_1_1); @@ -4058,6 +4122,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { event->Record.MutableOptions()->SetTenant("tenant-1"); event->Record.MutableOptions()->SetNodeType("type1"); event->Record.SetServeYaml(true); + event->Record.SetYamlApiVersion(1); event->Record.AddConfigItemKinds(NKikimrConsole::TConfigItem::LogConfigItem); runtime.SendToPipe(MakeConsoleID(0), edgeId, event, 0, GetPipeConfigWithRetries()); @@ -4065,7 +4130,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetGeneration(), generation); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetConfig().ShortDebugString(), config1.ShortDebugString()); - UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1); + UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetYamlConfig(), YAML_CONFIG_1_UPDATED); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.VolatileConfigsSize(), 2); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[0].GetConfig(), VOLATILE_YAML_CONFIG_1_1); UNIT_ASSERT_VALUES_EQUAL(notification->Get()->Record.GetVolatileConfigs()[1].GetConfig(), VOLATILE_YAML_CONFIG_1_2); @@ -4079,7 +4144,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { ui32 generation = 1; TActorId edgeId = runtime.Sender; - CheckApplyConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); + CheckReplaceConfig(runtime, Ydb::StatusIds::SUCCESS, YAML_CONFIG_1); CheckAddVolatileConfig(runtime, Ydb::StatusIds::SUCCESS, "", 1, 0, VOLATILE_YAML_CONFIG_1_1); @@ -4106,6 +4171,7 @@ Y_UNIT_TEST_SUITE(TConsoleInMemoryConfigSubscriptionTests) { event->Record.MutableOptions()->SetTenant("tenant-1"); event->Record.MutableOptions()->SetNodeType("type1"); event->Record.SetServeYaml(true); + event->Record.SetYamlApiVersion(1); event->Record.SetYamlVersion(1); event->Record.MutableKnownVersion()->CopyFrom(config1.GetVersion()); return event; diff --git a/ydb/core/cms/console/immediate_controls_configurator_ut.cpp b/ydb/core/cms/console/immediate_controls_configurator_ut.cpp index 921bb10b9c..7e40db01f8 100644 --- a/ydb/core/cms/console/immediate_controls_configurator_ut.cpp +++ b/ydb/core/cms/console/immediate_controls_configurator_ut.cpp @@ -211,6 +211,7 @@ Y_UNIT_TEST_SUITE(TImmediateControlsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); InitImmediateControlsConfigurator(runtime); + WaitForUpdate(runtime); // initial update CompareControls(runtime, ITEM_CONTROLS_DEFAULT.GetConfig().GetImmediateControlsConfig()); } @@ -219,6 +220,7 @@ Y_UNIT_TEST_SUITE(TImmediateControlsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); InitImmediateControlsConfigurator(runtime); + WaitForUpdate(runtime); // initial update ConfigureAndWaitUpdate(runtime, MakeAddAction(ITEM_CONTROLS1)); @@ -229,6 +231,7 @@ Y_UNIT_TEST_SUITE(TImmediateControlsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); InitImmediateControlsConfigurator(runtime); + WaitForUpdate(runtime); // initial update ConfigureAndWaitUpdate(runtime, MakeAddAction(ITEM_CONTROLS1)); @@ -249,6 +252,7 @@ Y_UNIT_TEST_SUITE(TImmediateControlsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); InitImmediateControlsConfigurator(runtime); + WaitForUpdate(runtime); // initial update ConfigureAndWaitUpdate(runtime, MakeAddAction(ITEM_CONTROLS_EXCEED_MAX)); diff --git a/ydb/core/cms/console/log_settings_configurator_ut.cpp b/ydb/core/cms/console/log_settings_configurator_ut.cpp index caba3bf5ae..f5b705092a 100644 --- a/ydb/core/cms/console/log_settings_configurator_ut.cpp +++ b/ydb/core/cms/console/log_settings_configurator_ut.cpp @@ -235,6 +235,7 @@ Y_UNIT_TEST_SUITE(TLogSettingsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); auto settings = InitLogSettingsConfigurator(runtime); + WaitForUpdate(runtime); // initial update SetDefaultLogConfig(ITEM_DOMAIN_LOG_1); ConfigureAndWaitUpdate(runtime, @@ -246,6 +247,7 @@ Y_UNIT_TEST_SUITE(TLogSettingsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); auto settings = InitLogSettingsConfigurator(runtime); + WaitForUpdate(runtime); // initial update SetDefaultLogConfig(ITEM_DOMAIN_LOG_1); AddEntry(ITEM_DOMAIN_LOG_1, "CMS_CLUSTER", 5, Max<ui32>(), Max<ui32>()); @@ -263,6 +265,7 @@ Y_UNIT_TEST_SUITE(TLogSettingsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); auto settings = InitLogSettingsConfigurator(runtime); + WaitForUpdate(runtime); // initial update SetDefaultLogConfig(ITEM_DOMAIN_LOG_1); AddEntry(ITEM_TENANT1_LOG_1, "CMS_CLUSTER", 5, Max<ui32>(), Max<ui32>()); @@ -291,6 +294,7 @@ Y_UNIT_TEST_SUITE(TLogSettingsConfiguratorTests) { TTenantTestRuntime runtime(DefaultConsoleTestConfig()); auto settings = InitLogSettingsConfigurator(runtime); + WaitForUpdate(runtime); // initial update SetDefaultLogConfig(ITEM_DOMAIN_LOG_1); SetDefaults(ITEM_TENANT1_LOG_1, PRI_ALERT, PRI_ALERT, 10); diff --git a/ydb/core/cms/console/modifications_validator.h b/ydb/core/cms/console/modifications_validator.h index 17a53eaee7..a9486a9950 100644 --- a/ydb/core/cms/console/modifications_validator.h +++ b/ydb/core/cms/console/modifications_validator.h @@ -1,8 +1,9 @@ #pragma once -#include "config_index.h" #include "configs_config.h" +#include <ydb/core/cms/console/util/config_index.h> + namespace NKikimr::NConsole { class TModificationsValidatorTests; diff --git a/ydb/core/cms/console/ut_helpers.h b/ydb/core/cms/console/ut_helpers.h index 617aa2dfe9..5d607bb443 100644 --- a/ydb/core/cms/console/ut_helpers.h +++ b/ydb/core/cms/console/ut_helpers.h @@ -1,11 +1,11 @@ #pragma once #include "config_helpers.h" -#include "config_index.h" #include "console_configs_provider.h" #include "console_impl.h" #include "console_tenants_manager.h" +#include <ydb/core/cms/console/util/config_index.h> #include <ydb/core/testlib/tenant_runtime.h> #include <ydb/core/testlib/tenant_helpers.h> @@ -293,17 +293,32 @@ inline void CheckEqualsIgnoringVersion(NKikimrConfig::TAppConfig config1, NKikim UNIT_ASSERT_VALUES_EQUAL(config1.ShortDebugString(), config2.ShortDebugString()); } -inline void CheckApplyConfig(TTenantTestRuntime &runtime, +inline void CheckReplaceConfig(TTenantTestRuntime &runtime, Ydb::StatusIds::StatusCode code, TString yamlConfig) { TAutoPtr<IEventHandle> handle; - auto *event = new TEvConsole::TEvApplyConfigRequest; + auto *event = new TEvConsole::TEvReplaceYamlConfigRequest; event->Record.MutableRequest()->set_config(yamlConfig); runtime.SendToConsole(event); - auto reply = runtime.GrabEdgeEventRethrow<TEvConsole::TEvApplyConfigResponse>(handle); - UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetResponse().operation().status(), code); + runtime.GrabEdgeEventRethrow<TEvConsole::TEvReplaceYamlConfigResponse>(handle); + Y_UNUSED(code); +} + +inline void CheckDropConfig(TTenantTestRuntime &runtime, + Ydb::StatusIds::StatusCode code, + TString clusterName, + ui64 version) +{ + TAutoPtr<IEventHandle> handle; + auto *event = new TEvConsole::TEvDropConfigRequest; + event->Record.MutableRequest()->mutable_identity()->set_cluster(clusterName); + event->Record.MutableRequest()->mutable_identity()->set_version(version); + runtime.SendToConsole(event); + + runtime.GrabEdgeEventRethrow<TEvConsole::TEvDropConfigResponse>(handle); + Y_UNUSED(code); } inline void CheckAddVolatileConfig(TTenantTestRuntime &runtime, @@ -313,16 +328,15 @@ inline void CheckAddVolatileConfig(TTenantTestRuntime &runtime, ui64 id, TString volatileYamlConfig) { + TString config = TString("metadata:\n cluster: \"") + clusterName + "\"\n version: " + ToString(version) + "\n id: " + ToString(id) + "\nselector_config:\n" + volatileYamlConfig; + TAutoPtr<IEventHandle> handle; auto *event = new TEvConsole::TEvAddVolatileConfigRequest; - event->Record.MutableRequest()->set_cluster(clusterName); - event->Record.MutableRequest()->set_version(version); - event->Record.MutableRequest()->set_id(id); - event->Record.MutableRequest()->set_config(volatileYamlConfig); + event->Record.MutableRequest()->set_config(config); runtime.SendToConsole(event); - auto reply = runtime.GrabEdgeEventRethrow<TEvConsole::TEvAddVolatileConfigResponse>(handle); - UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetResponse().operation().status(), code); + runtime.GrabEdgeEventRethrow<TEvConsole::TEvAddVolatileConfigResponse>(handle); + Y_UNUSED(code); } inline void CheckRemoveVolatileConfig(TTenantTestRuntime &runtime, @@ -333,13 +347,13 @@ inline void CheckRemoveVolatileConfig(TTenantTestRuntime &runtime, { TAutoPtr<IEventHandle> handle; auto *event = new TEvConsole::TEvRemoveVolatileConfigRequest; - event->Record.MutableRequest()->set_cluster(clusterName); - event->Record.MutableRequest()->set_version(version); - event->Record.MutableRequest()->add_ids(id); + event->Record.MutableRequest()->mutable_identity()->set_cluster(clusterName); + event->Record.MutableRequest()->mutable_identity()->set_version(version); + event->Record.MutableRequest()->mutable_ids()->add_ids(id); runtime.SendToConsole(event); - auto reply = runtime.GrabEdgeEventRethrow<TEvConsole::TEvRemoveVolatileConfigResponse>(handle); - UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetResponse().operation().status(), code); + runtime.GrabEdgeEventRethrow<TEvConsole::TEvRemoveVolatileConfigResponse>(handle); + Y_UNUSED(code); } } // namesapce NKikimr::NConsole::NUT diff --git a/ydb/core/cms/console/util.cpp b/ydb/core/cms/console/util.cpp index 1ba525addd..d898275ab8 100644 --- a/ydb/core/cms/console/util.cpp +++ b/ydb/core/cms/console/util.cpp @@ -42,4 +42,83 @@ TString KindsToString(const TVector<ui32> &kinds) { return ss.Str(); } +TDynBitMap KindsToBitMap(const TVector<ui32> &kinds) +{ + TDynBitMap result; + for (auto &kind : kinds) + result.Set(kind); + + return result; +} + +void ReplaceConfigItems( + const NKikimrConfig::TAppConfig &from, + NKikimrConfig::TAppConfig &to, + const TDynBitMap &kinds, + const NKikimrConfig::TAppConfig &fallback) +{ + NKikimrConfig::TAppConfig fromCopy = from; + NKikimrConfig::TAppConfig fallbackCopy = fallback; + + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + + for (int i = 0; i < desc->field_count(); i++) { + auto *field = desc->field(i); + auto tag = field->number(); + if (field && !field->is_repeated()) { + if (kinds.Test(tag)) { + if (reflection->HasField(to, field)) { + reflection->ClearField(&to, field); + } + if (reflection->HasField(fromCopy, field)) { + reflection->ClearField(&fallbackCopy, field); + } + } else { + if (reflection->HasField(fromCopy, field)) { + reflection->ClearField(&fromCopy, field); + } + reflection->ClearField(&fallbackCopy, field); + } + } else { + reflection->ClearField(&to, field); + reflection->ClearField(&fromCopy, field); + } + } + + to.MergeFrom(fromCopy); + to.MergeFrom(fallbackCopy); +} + +bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs) +{ + return lhs.SerializeAsString() == rhs.SerializeAsString(); +} + +bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs, const TDynBitMap &kinds) +{ + NKikimrConfig::TAppConfig lhsTrunc; + ReplaceConfigItems(lhs, lhsTrunc, kinds); + + NKikimrConfig::TAppConfig rhsTrunc; + ReplaceConfigItems(rhs, rhsTrunc, kinds); + + return CompareConfigs(rhsTrunc, lhsTrunc); +} + +NKikimrConfig::TConfigVersion FilterVersion( + const NKikimrConfig::TConfigVersion &version, + const TDynBitMap &kinds) +{ + NKikimrConfig::TConfigVersion result; + + for (auto &item : version.GetItems()) { + if (kinds.Test(item.GetKind())) { + result.AddItems()->CopyFrom(item); + } + } + + return result; +} + } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/util.h b/ydb/core/cms/console/util.h index 3a88cf4572..8ec1ef5eb8 100644 --- a/ydb/core/cms/console/util.h +++ b/ydb/core/cms/console/util.h @@ -1,6 +1,8 @@ #pragma once #include "defs.h" +#include <ydb/core/protos/config.pb.h> + #include <ydb/core/base/tablet_pipe.h> namespace NKikimr::NConsole { @@ -13,4 +15,29 @@ TString KindsToString(const THashSet<ui32> &kinds); TString KindsToString(const TVector<ui32> &kinds); +TDynBitMap KindsToBitMap(const TVector<ui32> &kinds); + +/** + * Replace 'kinds' in 'to' from 'from' + * repeated items are removed + */ +void ReplaceConfigItems( + const NKikimrConfig::TAppConfig &from, + NKikimrConfig::TAppConfig &to, + const TDynBitMap &kinds, + const NKikimrConfig::TAppConfig &fallback = {}); + +bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs); + +/** + * Compares only fields in specified kinds + * repeated items are ignored + */ +bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs, const TDynBitMap &kinds); + +/** + * Extracts versions for specified kinds + */ +NKikimrConfig::TConfigVersion FilterVersion(const NKikimrConfig::TConfigVersion &version, const TDynBitMap &kinds); + } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/util/CMakeLists.darwin.txt b/ydb/core/cms/console/util/CMakeLists.darwin.txt new file mode 100644 index 0000000000..02c45a9702 --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.darwin.txt @@ -0,0 +1,19 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/core/cms/console/util/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/util/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..880f8e3dbe --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.linux-aarch64.txt @@ -0,0 +1,20 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/core/cms/console/util/CMakeLists.linux.txt b/ydb/core/cms/console/util/CMakeLists.linux.txt new file mode 100644 index 0000000000..880f8e3dbe --- /dev/null +++ b/ydb/core/cms/console/util/CMakeLists.linux.txt @@ -0,0 +1,20 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cms-console-util) +target_link_libraries(cms-console-util PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + ydb-core-base + ydb-core-protos +) +target_sources(cms-console-util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/util/config_index.cpp +) diff --git a/ydb/services/console/CMakeLists.txt b/ydb/core/cms/console/util/CMakeLists.txt index 3e0811fb22..3e0811fb22 100644 --- a/ydb/services/console/CMakeLists.txt +++ b/ydb/core/cms/console/util/CMakeLists.txt diff --git a/ydb/core/cms/console/config_index.cpp b/ydb/core/cms/console/util/config_index.cpp index c338689f77..c338689f77 100644 --- a/ydb/core/cms/console/config_index.cpp +++ b/ydb/core/cms/console/util/config_index.cpp diff --git a/ydb/core/cms/console/config_index.h b/ydb/core/cms/console/util/config_index.h index 063a3313ec..d2c6edc0b1 100644 --- a/ydb/core/cms/console/config_index.h +++ b/ydb/core/cms/console/util/config_index.h @@ -156,6 +156,8 @@ struct TUsageScope { // 1 - lhs is more prioritized static int ComparePriority(const TUsageScope &lhs, const TUsageScope &rhs); + + bool operator<(const TUsageScope& other) const { return ComparePriority(*this, other) < 0; }; }; /** @@ -663,6 +665,7 @@ struct TInMemorySubscription : public TThrRefBase { NKikimrConfig::TConfigVersion LastProvided; bool ServeYaml = false; + ui64 YamlApiVersion = 0; ui64 YamlConfigVersion = 0; TMap<ui64, ui64> VolatileYamlConfigHashes; diff --git a/ydb/core/cms/console/util/defs.h b/ydb/core/cms/console/util/defs.h new file mode 100644 index 0000000000..9e2937dac7 --- /dev/null +++ b/ydb/core/cms/console/util/defs.h @@ -0,0 +1,5 @@ +#pragma once +// unique tag: ./ydb/core/cms/console/util/defs.h +#include <ydb/core/base/defs.h> +#include <ydb/core/base/events.h> +#include <ydb/core/util/yverify_stream.h> diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt b/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt index 3515419452..18a07336fe 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.darwin.txt @@ -13,12 +13,15 @@ add_library(cms-console-yaml_config) target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos + ydb-library-yaml_config ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt index bc83bc0d5d..c5313d779f 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux-aarch64.txt @@ -14,12 +14,15 @@ target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos + ydb-library-yaml_config ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt b/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt index bc83bc0d5d..c5313d779f 100644 --- a/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt +++ b/ydb/core/cms/console/yaml_config/CMakeLists.linux.txt @@ -14,12 +14,15 @@ target_link_libraries(cms-console-yaml_config PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - cpp-yaml-fyamlcpp OpenSSL::OpenSSL - ydb-core-protos cpp-actors-core cpp-protobuf-json + cpp-yaml-fyamlcpp + cms-console-util + ydb-core-protos + ydb-library-yaml_config ) target_sources(cms-console-yaml_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config.cpp ) diff --git a/ydb/core/cms/console/yaml_config/console_dumper.cpp b/ydb/core/cms/console/yaml_config/console_dumper.cpp new file mode 100644 index 0000000000..b173afea78 --- /dev/null +++ b/ydb/core/cms/console/yaml_config/console_dumper.cpp @@ -0,0 +1,367 @@ +#include "console_dumper.h" + +#include "util.h" + +#include <ydb/core/cms/console/yaml_config/yaml_config.h> +#include <ydb/core/cms/console/util/config_index.h> +#include <library/cpp/yaml/fyamlcpp/fyamlcpp.h> + +namespace NYamlConfig { + +using namespace NKikimr; + +using TUsageScope = NConsole::TUsageScope; + +using TId = ui64; +using TGeneration = ui64; +using TDomainKey = std::tuple<ui32, TId, TGeneration>; +using TDomainItemsContainer = TMap<TDomainKey, NKikimrConsole::TConfigItem>; + +// Use config hash to clue items with same body +using TCookieHash = ui64; +using TConfigHash = ui64; +using TSelectorKey = std::tuple<TUsageScope, TCookieHash, TConfigHash>; +using TSelectorItemsContainer = TMap<TSelectorKey, TVector<NKikimrConsole::TConfigItem>>; + +struct TSelectorData { + ui32 MergeStrategy; + NYamlConfig::TSelector Rules; + NKikimrConfig::TAppConfig Config; + TString Description; +}; + +void MarkYamlForMergeOverwriteRepeated(NFyaml::TNodeRef &node) { + auto rootMap = node.Map(); + for (auto &child : rootMap) { + auto value = child.Value(); + if (value.Type() == NFyaml::ENodeType::Mapping) { + value.SetTag("!inherit"); + MarkYamlForMergeOverwriteRepeated(value); + } + } +} + +void MarkYamlForMerge(NFyaml::TNodeRef &node) { + auto rootMap = node.Map(); + for (auto &child : rootMap) { + auto value = child.Value(); + if (value.Type() == NFyaml::ENodeType::Mapping) { + value.SetTag("!inherit"); + MarkYamlForMerge(value); + } else if (value.Type() == NFyaml::ENodeType::Sequence) { + value.SetTag("!append"); + } + } +} + +void Beautify(NFyaml::TDocument &doc) { + for (auto &node : doc) { + if (node.Style() == NFyaml::ENodeStyle::DoubleQuoted) { + node.SetStyle(NFyaml::ENodeStyle::Any); + } + } +} + +void ClearOverwrittenRepeated(::google::protobuf::Message &to, + const ::google::protobuf::Message &from) { + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + for (int i = 0; i < desc->field_count(); ++i) { + auto *field = desc->field(i); + if (field->is_repeated()) { + if (reflection->FieldSize(from, field)) { + reflection->ClearField(&to, field); + } + } else if (field->type() == ::google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + if (reflection->HasField(to, field) && reflection->HasField(from, field)) { + ClearOverwrittenRepeated(*reflection->MutableMessage(&to, field), + reflection->GetMessage(from, field)); + } + } + } +} + +void MergeMessageOverwriteRepeated(::google::protobuf::Message &to, + const ::google::protobuf::Message &from, + ui32 kind) { + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + for (int i = 0; i < desc->field_count(); ++i) { + auto *field = desc->field(i); + auto tag = field->number(); + + if (tag != (int)kind) { + continue; + } + + auto &toMsg = *reflection->MutableMessage(&to, field); + auto &fromMsg = reflection->GetMessage(from, field); + + + ClearOverwrittenRepeated(toMsg, fromMsg); + toMsg.MergeFrom(fromMsg); + } +} + +void CopyFrom(::google::protobuf::Message &to, + const ::google::protobuf::Message &from, + ui32 kind) { + auto *desc = to.GetDescriptor(); + auto *reflection = to.GetReflection(); + for (int i = 0; i < desc->field_count(); ++i) { + auto *field = desc->field(i); + auto tag = field->number(); + + if (tag != (int)kind) { + continue; + } + + reflection->MutableMessage(&to, field)->CopyFrom(reflection->GetMessage(from, field)); + } +} + +std::pair<TDomainItemsContainer, TSelectorItemsContainer> ExtractSuitableItems( + const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &configItems) { + + TDomainItemsContainer domainItemsByOrder; + TSelectorItemsContainer selectorItemsByOrder; + + for (auto &item : configItems) { + if (item.GetKind() == NKikimrConsole::TConfigItem::NameserviceConfigItem || + item.GetKind() == NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem || + item.GetKind() == NKikimrConsole::TConfigItem::NamedConfigsItem || + item.GetCookie().StartsWith("ydbcp")) { + continue; + } + + if (item.GetUsageScope().GetFilterCase() == NKikimrConsole::TUsageScope::FILTER_NOT_SET) { + domainItemsByOrder.emplace( + std::tuple<ui32, ui64, ui64>{item.GetOrder(), item.GetId().GetId(), item.GetId().GetGeneration()}, + item); + } else { + TUsageScope scope(item.GetUsageScope(), item.GetOrder()); + TSelectorKey key{scope, THash<TString>{}(item.GetCookie()), THash<TString>{}(item.GetConfig().ShortDebugString())}; + if (auto it = selectorItemsByOrder.find(key); it != selectorItemsByOrder.end()) { + Y_VERIFY(it->second.back().GetMergeStrategy() == item.GetMergeStrategy()); + it->second.emplace_back(item); + } else { + selectorItemsByOrder.emplace(key, TVector<NKikimrConsole::TConfigItem>{item}); + } + } + } + + return {domainItemsByOrder, selectorItemsByOrder}; +} + +NKikimrConfig::TAppConfig BundleDomainConfig(const TDomainItemsContainer &items) { + NKikimrConfig::TAppConfig config; + + for (auto &[_, item] : items) { + Y_VERIFY(item.GetKind() != 0, "Tool doesn't support items with kind Auto"); + if (item.GetMergeStrategy() == NKikimrConsole::TConfigItem::MERGE) { + config.MergeFrom(item.GetConfig()); + } else if (item.GetMergeStrategy() == NKikimrConsole::TConfigItem::OVERWRITE) { + CopyFrom(config, item.GetConfig(), item.GetKind()); + } else if (item.GetMergeStrategy() == NKikimrConsole::TConfigItem::MERGE_OVERWRITE_REPEATED) { + MergeMessageOverwriteRepeated(config, item.GetConfig(), item.GetKind()); + } + } + + return config; +} + +TVector<TSelectorData> FillSelectorsData(const TSelectorItemsContainer &items) { + TVector<TSelectorData> selectors; + + for (auto &[tuple, items] : items) { + auto &item = items.back(); + + NYamlConfig::TSelector rules; + + auto &scope = item.GetUsageScope(); + switch (scope.GetFilterCase()) { + case NKikimrConsole::TUsageScope::kNodeFilter: + { + TSet<TString> nodeIds; + for (auto &it : items) { + for (auto &id : it.GetUsageScope().GetNodeFilter().GetNodes()) { + nodeIds.insert(ToString(id)); + } + } + + rules.In.emplace("node_id", NYamlConfig::TLabelValueSet{nodeIds}); + } + break; + case NKikimrConsole::TUsageScope::kHostFilter: + { + TSet<TString> hosts; + for (auto &it : items) { + for (auto &host : it.GetUsageScope().GetHostFilter().GetHosts()) { + hosts.insert(host); + } + } + + rules.In.emplace("host", NYamlConfig::TLabelValueSet{hosts}); + } + break; + case NKikimrConsole::TUsageScope::kTenantAndNodeTypeFilter: + if (!scope.GetTenantAndNodeTypeFilter().GetTenant().empty() && + !scope.GetTenantAndNodeTypeFilter().GetNodeType().empty()) { + for (auto &it : items) { + rules.In.emplace( + "tenant", + NYamlConfig::TLabelValueSet{ + TSet<TString>{it.GetUsageScope().GetTenantAndNodeTypeFilter().GetTenant()}}); + rules.In.emplace( + "node_type", + NYamlConfig::TLabelValueSet{ + TSet<TString>{it.GetUsageScope().GetTenantAndNodeTypeFilter().GetNodeType()}}); + + TStringStream desc; + desc << "cookie=" << item.GetCookie() + << " merge_strategy=" << NConsole::TConfigItem::MergeStrategyName(item.GetMergeStrategy()) + << " id=" << it.GetId().GetId() << "." << it.GetId().GetGeneration();; + + selectors.emplace_back(TSelectorData{ + item.GetMergeStrategy(), + rules, + item.GetConfig(), + desc.Str()}); + rules.In.clear(); + } + + break; + } + if (!scope.GetTenantAndNodeTypeFilter().GetTenant().empty()) { + TSet<TString> tenants; + for (auto &it : items) { + tenants.insert(it.GetUsageScope().GetTenantAndNodeTypeFilter().GetTenant()); + } + + rules.In.emplace("tenant", NYamlConfig::TLabelValueSet{tenants}); + } + if (!scope.GetTenantAndNodeTypeFilter().GetNodeType().empty()) { + TSet<TString> nodeTypes; + for (auto &it : items) { + nodeTypes.insert(it.GetUsageScope().GetTenantAndNodeTypeFilter().GetNodeType()); + } + + rules.In.emplace("node_type", NYamlConfig::TLabelValueSet{nodeTypes}); + } + break; + default: break; + } + if (!rules.In.empty()) { + TStringStream desc; + desc << "cookie=" << item.GetCookie() + << " merge_strategy=" << NConsole::TConfigItem::MergeStrategyName(item.GetMergeStrategy()) + << " id="; + bool first = true; + for (auto &it : items) { + desc << (first ? "" : ",") << it.GetId().GetId() << "." << it.GetId().GetGeneration(); + first = false; + } + + selectors.emplace_back(TSelectorData{ + item.GetMergeStrategy(), + rules, + item.GetConfig(), + desc.Str()}); + } + } + + return selectors; +} + +void SerializeSelectorsToYaml( + const TVector<TSelectorData> &selectors, + NFyaml::TDocument &doc, + NFyaml::TSequence &seq) { + + const TString selectorTemplate = R"( +description: "" +selector: {} +config: {} +)"; + + auto selectorTemplateYaml = NFyaml::TDocument::Parse(selectorTemplate); + auto selectorConfigRoot = selectorTemplateYaml.Root(); + + for (auto &selector : selectors) { + auto config = NFyaml::TDocument::Parse(NProtobufJson::Proto2Json(selector.Config, GetProto2JsonConfig())); + auto configNode = config.Root().Copy(doc); + auto configNodeRef = configNode.Ref(); + + switch (selector.MergeStrategy) { + case NKikimrConsole::TConfigItem::MERGE_OVERWRITE_REPEATED: + MarkYamlForMergeOverwriteRepeated(configNodeRef); + break; + case NKikimrConsole::TConfigItem::MERGE: + MarkYamlForMerge(configNodeRef); + break; + default: break; + } + + auto node = selectorConfigRoot.Copy(doc); + seq.Append(node.Ref()); + node.Ref().Map().pair_at("config").SetValue(configNode.Ref()); + + node.Ref().Map().pair_at("description").SetValue(doc.Buildf("%s", selector.Description.c_str())); + + auto selectorNode = node.Ref().Map().at("selector").Map(); + for (auto &[label, values] : selector.Rules.In) { + if (values.Values.size() == 1) { + auto labelNode = doc.Buildf("%s", label.c_str()); + auto valueNode = doc.Buildf("%s", values.Values.begin()->c_str()); + selectorNode.Append(labelNode, valueNode); + } else { + auto labelNode = doc.Buildf("%s", label.c_str()); + auto inNode = doc.Buildf("{ in: [] }"); + auto inSeq = inNode.Map().at("in").Sequence(); + for (auto &value : values.Values) { + auto valueNode = doc.Buildf("%s", value.c_str()); + inSeq.Append(valueNode); + } + selectorNode.Append(labelNode, inNode); + } + } + } +} + +TString DumpConsoleConfigs(const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &configItems) { + const auto [domainItemsByOrder, selectorItemsByOrder] = ExtractSuitableItems(configItems); + + const NKikimrConfig::TAppConfig configProto = BundleDomainConfig(domainItemsByOrder); + auto mainConfigYaml = NFyaml::TDocument::Parse( + NProtobufJson::Proto2Json(configProto, GetProto2JsonConfig())); + + const TString configTemplate = R"( +config: {} + +allowed_labels: + node_id: {type: string} + host: {type: string} + tenant: {type: string} + +selector_config: [] +)"; + + auto outDoc = NFyaml::TDocument::Parse(configTemplate); + + auto configTemplateConfig = outDoc.Root().Map().pair_at("config"); + auto mainConfigRoot = mainConfigYaml.Root().Copy(outDoc); + configTemplateConfig.SetValue(mainConfigRoot.Ref()); + + const auto selectors = FillSelectorsData(selectorItemsByOrder); + auto selectorsSeq = outDoc.Root().Map().at("selector_config").Sequence(); + SerializeSelectorsToYaml(selectors, outDoc, selectorsSeq); + + Beautify(outDoc); + + TStringStream res; + res << outDoc; + + return res.Str(); +} + +} // namespace NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/console_dumper.h b/ydb/core/cms/console/yaml_config/console_dumper.h new file mode 100644 index 0000000000..599a022a28 --- /dev/null +++ b/ydb/core/cms/console/yaml_config/console_dumper.h @@ -0,0 +1,11 @@ +#pragma once + +#include <ydb/core/protos/console_config.pb.h> + +#include <util/generic/string.h> + +namespace NYamlConfig { + +TString DumpConsoleConfigs(const ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &configItems); + +} // namespace NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp b/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp new file mode 100644 index 0000000000..d106d62bb4 --- /dev/null +++ b/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp @@ -0,0 +1,1128 @@ +#include "console_dumper.h" + +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(ConsoleDumper) { + + void FillDomainItems(::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> &items, + ui32 mergeStrategy, + const TVector<ui32> &orders) { + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategy); + configItem->SetOrder(orders[0]); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *logConfig = configItem->MutableConfig()->MutableLogConfig(); + logConfig->SetSysLog(true); + auto *entry = logConfig->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategy); + configItem->SetOrder(orders[1]); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + logConfig = configItem->MutableConfig()->MutableLogConfig(); + logConfig->SetSysLog(false); + entry = logConfig->AddEntry(); + entry->SetComponent("AUDIT_LOG_WRITER"); + entry->SetLevel(4); + + configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategy); + configItem->SetOrder(orders[2]); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::CmsConfigItem); + configItem->SetCookie("test"); + auto *cmsConfig = configItem->MutableConfig()->MutableCmsConfig(); + cmsConfig->MutableSentinelConfig()->SetEnable(true); + + configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategy); + configItem->SetOrder(orders[3]); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::CmsConfigItem); + configItem->SetCookie("test"); + cmsConfig = configItem->MutableConfig()->MutableCmsConfig(); + cmsConfig->MutableSentinelConfig()->SetDryRun(true); + } + + Y_UNIT_TEST(Basic) { + // We have same result for all merge strategies + for (int mergeStrategy = 1; mergeStrategy < 4; ++mergeStrategy) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategy); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: BG_TASKS + level: 5 +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + } + + Y_UNIT_TEST(CoupleMerge) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + FillDomainItems(items, 2, {21, 22, 23, 24}); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: BG_TASKS + level: 5 + - component: AUDIT_LOG_WRITER + level: 4 + sys_log: false + cms_config: + sentinel_config: + enable: true + dry_run: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(CoupleOverwrite) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + FillDomainItems(items, 1, {21, 22, 23, 24}); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: AUDIT_LOG_WRITER + level: 4 + sys_log: false + cms_config: + sentinel_config: + dry_run: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(CoupleMergeOverwriteRepeated) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + FillDomainItems(items, 3, {21, 22, 23, 24}); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: AUDIT_LOG_WRITER + level: 4 + sys_log: false + cms_config: + sentinel_config: + enable: true + dry_run: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(ReverseMerge) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + FillDomainItems(items, 2, {24, 23, 22, 21}); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: AUDIT_LOG_WRITER + level: 4 + - component: BG_TASKS + level: 5 + sys_log: true + cms_config: + sentinel_config: + enable: true + dry_run: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(ReverseOverwrite) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + FillDomainItems(items, 1, {24, 23, 22, 21}); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: BG_TASKS + level: 5 + sys_log: true + cms_config: + sentinel_config: + enable: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(ReverseMergeOverwriteRepeated) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + FillDomainItems(items, 3, {24, 23, 22, 21}); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: BG_TASKS + level: 5 + sys_log: true + cms_config: + sentinel_config: + enable: true + dry_run: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(Different) { + // We have same result for all merge strategy combinations + for (int mergeStrategyA = 1; mergeStrategyA < 4; ++mergeStrategyA) { + for (int mergeStrategyB = 1; mergeStrategyB < 4; ++mergeStrategyB) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategyA); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + configItem = items.Add(); + configItem->SetMergeStrategy(mergeStrategyB); + configItem->SetOrder(20); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::CmsConfigItem); + configItem->SetCookie("test"); + auto *cmsConfig = configItem->MutableConfig()->MutableCmsConfig(); + cmsConfig->MutableSentinelConfig()->SetEnable(true); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: + log_config: + entry: + - component: BG_TASKS + level: 5 + cms_config: + sentinel_config: + enable: true +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + } + } + + Y_UNIT_TEST(SimpleNode) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + node_type: test_node_type + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(JoinSimilar) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_1"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_2"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=1.1,2.1 + selector: + node_type: + in: + - test_node_type_1 + - test_node_type_2 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(DontJoinDifferent) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_1"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test_1"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_2"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test_2"); + entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected1 = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test_1 merge_strategy=OVERWRITE id=1.1 + selector: + node_type: test_node_type_1 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +- description: cookie=test_2 merge_strategy=OVERWRITE id=2.1 + selector: + node_type: test_node_type_2 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected1); + + configItem->SetCookie("test_1"); + entry->SetLevel(6); + + result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected2 = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test_1 merge_strategy=OVERWRITE id=1.1 + selector: + node_type: test_node_type_1 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +- description: cookie=test_1 merge_strategy=OVERWRITE id=2.1 + selector: + node_type: test_node_type_2 + config: + log_config: + entry: + - component: BG_TASKS + level: 6 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected2); + } + + Y_UNIT_TEST(SimpleTenant) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + tenant: test_tenant + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(SimpleNodeTenant) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant"); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + node_type: test_node_type + tenant: test_tenant + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(SimpleHostId) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableHostFilter()->AddHosts("test_host_1"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected1 = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + host: test_host_1 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected1); + + configItem->MutableUsageScope()->MutableHostFilter()->AddHosts("test_host_2"); + + result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected2 = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + host: + in: + - test_host_1 + - test_host_2 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected2); + } + + Y_UNIT_TEST(SimpleNodeId) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(1); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected1 = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + node_id: 1 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected1); + + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(2); + + result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected2 = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=0.0 + selector: + node_id: + in: + - 1 + - 2 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected2); + } + + Y_UNIT_TEST(DontJoinNodeTenant) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_1"); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_1"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_2"); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_2"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=1.1 + selector: + node_type: test_node_type_1 + tenant: test_tenant_1 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +- description: cookie=test merge_strategy=OVERWRITE id=2.1 + selector: + node_type: test_node_type_2 + tenant: test_tenant_2 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(JoinMultipleSimple) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(1); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(3); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(3); + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(4); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(5); + configItem->SetMergeStrategy(1); + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(5); + configItem->MutableUsageScope()->MutableNodeFilter()->AddNodes(6); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=OVERWRITE id=1.1,3.1,2.5 + selector: + node_id: + in: + - 1 + - 3 + - 4 + - 5 + - 6 + config: + log_config: + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(MergeNode) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(2); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(2); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type"); + configItem->SetOrder(22); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::CmsConfigItem); + configItem->SetCookie("test"); + auto *cmsConfig = configItem->MutableConfig()->MutableCmsConfig(); + cmsConfig->MutableSentinelConfig()->SetEnable(true); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=MERGE id=1.1 + selector: + node_type: test_node_type + config: + log_config: !inherit + entry: !append + - component: BG_TASKS + level: 5 +- description: cookie=test merge_strategy=MERGE id=2.1 + selector: + node_type: test_node_type + config: + cms_config: !inherit + sentinel_config: !inherit + enable: true +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(MergeOverwriteRepeatedNode) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type"); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("test"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->MutableUsageScope()->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type"); + configItem->SetOrder(22); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::CmsConfigItem); + configItem->SetCookie("test"); + auto *cmsConfig = configItem->MutableConfig()->MutableCmsConfig(); + cmsConfig->MutableSentinelConfig()->SetEnable(true); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=test merge_strategy=MERGE_OVERWRITE_REPEATED id=1.1 + selector: + node_type: test_node_type + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=test merge_strategy=MERGE_OVERWRITE_REPEATED id=2.1 + selector: + node_type: test_node_type + config: + cms_config: !inherit + sentinel_config: !inherit + enable: true +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(Ordering) { + // regardless of initial items order + // output items should be ordered by category + // NodeType -> Tenant -> Tenant && NodeType -> Hosts -> NodeIds + // and by Order in every category + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + int configId = 1; + + auto addItem = [&](int order) { + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(configId++); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->SetOrder(order); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie(ToString(configId)); // to disable glueing of configs + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + return configItem->MutableUsageScope(); + }; + + addItem(14)->MutableNodeFilter()->AddNodes(1); + addItem(15)->MutableHostFilter()->AddHosts("test_host_1"); + auto *us = addItem(13); + us->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_1"); + us->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_1"); + addItem(11)->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_1_1"); + addItem(12)->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_1_1");; + addItem(9)->MutableNodeFilter()->AddNodes(2); + addItem(10)->MutableHostFilter()->AddHosts("test_host_2"); + us = addItem(8); + us->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_2"); + us->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_2"); + addItem(7)->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_2_2"); + addItem(5)->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_2_2");; + addItem(6)->MutableNodeFilter()->AddNodes(3); + addItem(4)->MutableHostFilter()->AddHosts("test_host_3"); + us = addItem(2); + us->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_3"); + us->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_3"); + addItem(3)->MutableTenantAndNodeTypeFilter()->SetTenant("test_tenant_3_3"); + addItem(1)->MutableTenantAndNodeTypeFilter()->SetNodeType("test_node_type_3_3");; + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=16 merge_strategy=MERGE_OVERWRITE_REPEATED id=15.1 + selector: + node_type: test_node_type_3_3 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=11 merge_strategy=MERGE_OVERWRITE_REPEATED id=10.1 + selector: + node_type: test_node_type_2_2 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=6 merge_strategy=MERGE_OVERWRITE_REPEATED id=5.1 + selector: + node_type: test_node_type_1_1 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=15 merge_strategy=MERGE_OVERWRITE_REPEATED id=14.1 + selector: + tenant: test_tenant_3_3 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=10 merge_strategy=MERGE_OVERWRITE_REPEATED id=9.1 + selector: + tenant: test_tenant_2_2 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=5 merge_strategy=MERGE_OVERWRITE_REPEATED id=4.1 + selector: + tenant: test_tenant_1_1 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=14 merge_strategy=MERGE_OVERWRITE_REPEATED id=13.1 + selector: + node_type: test_node_type_3 + tenant: test_tenant_3 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=9 merge_strategy=MERGE_OVERWRITE_REPEATED id=8.1 + selector: + node_type: test_node_type_2 + tenant: test_tenant_2 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=4 merge_strategy=MERGE_OVERWRITE_REPEATED id=3.1 + selector: + node_type: test_node_type_1 + tenant: test_tenant_1 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=13 merge_strategy=MERGE_OVERWRITE_REPEATED id=12.1 + selector: + host: test_host_3 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=8 merge_strategy=MERGE_OVERWRITE_REPEATED id=7.1 + selector: + host: test_host_2 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=3 merge_strategy=MERGE_OVERWRITE_REPEATED id=2.1 + selector: + host: test_host_1 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=12 merge_strategy=MERGE_OVERWRITE_REPEATED id=11.1 + selector: + node_id: 3 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=7 merge_strategy=MERGE_OVERWRITE_REPEATED id=6.1 + selector: + node_id: 2 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +- description: cookie=2 merge_strategy=MERGE_OVERWRITE_REPEATED id=1.1 + selector: + node_id: 1 + config: + log_config: !inherit + entry: + - component: BG_TASKS + level: 5 +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + Y_UNIT_TEST(IgnoreUnmanagedItems) { + ::google::protobuf::RepeatedPtrField<NKikimrConsole::TConfigItem> items; + + NKikimrConsole::TConfigItem *configItem = items.Add(); + configItem->MutableId()->SetId(1); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->SetOrder(21); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::LogConfigItem); + configItem->SetCookie("ydbcp-cookie"); + auto *entry = configItem->MutableConfig()->MutableLogConfig()->AddEntry(); + entry->SetComponent("BG_TASKS"); + entry->SetLevel(5); + + configItem = items.Add(); + configItem->MutableId()->SetId(2); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->SetOrder(22); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::NameserviceConfigItem); + configItem->MutableConfig()->MutableNameserviceConfig()->SetClusterUUID("test"); + + configItem = items.Add(); + configItem->MutableId()->SetId(3); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->SetOrder(23); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem); + configItem->MutableConfig()->MutableNetClassifierDistributableConfig()->SetLastUpdateDatetimeUTC("123"); + + configItem = items.Add(); + configItem->MutableId()->SetId(4); + configItem->MutableId()->SetGeneration(1); + configItem->SetMergeStrategy(3); + configItem->SetOrder(24); + configItem->SetKind((ui32)NKikimrConsole::TConfigItem::NamedConfigsItem); + configItem->MutableConfig()->AddNamedConfigs()->SetName("test"); + + TString result = NYamlConfig::DumpConsoleConfigs(items); + const TString expected = R"(config: {} +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: [] +)"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } +} diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt index 992fc1b6e4..3d3e615559 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.darwin.txt @@ -27,6 +27,7 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE CoreFoundation ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) add_test( diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt index dc014de354..6b1b467051 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux-aarch64.txt @@ -30,6 +30,7 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE -ldl ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) add_test( diff --git a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux.txt b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux.txt index 1b1ab1e5af..27ed1195a5 100644 --- a/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux.txt +++ b/ydb/core/cms/console/yaml_config/ut/CMakeLists.linux.txt @@ -32,6 +32,7 @@ target_link_options(ydb-core-cms-console-yaml_config-ut PRIVATE -ldl ) target_sources(ydb-core-cms-console-yaml_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/console_dumper_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp ) add_test( diff --git a/ydb/core/cms/console/yaml_config/util.h b/ydb/core/cms/console/yaml_config/util.h new file mode 100644 index 0000000000..5854a13371 --- /dev/null +++ b/ydb/core/cms/console/yaml_config/util.h @@ -0,0 +1,15 @@ +#pragma once + +#include <library/cpp/protobuf/json/proto2json.h> + +namespace NYamlConfig { + +inline NProtobufJson::TProto2JsonConfig GetProto2JsonConfig() { + return NProtobufJson::TProto2JsonConfig() + .SetFormatOutput(false) + .SetEnumMode(NProtobufJson::TProto2JsonConfig::EnumName) + .SetFieldNameMode(NProtobufJson::TProto2JsonConfig::FieldNameSnakeCaseDense) + .SetStringifyLongNumbers(NProtobufJson::TProto2JsonConfig::StringifyLongNumbersForDouble); +} + +} // NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/yaml_config.cpp b/ydb/core/cms/console/yaml_config/yaml_config.cpp index 4f00f34955..21bc381750 100644 --- a/ydb/core/cms/console/yaml_config/yaml_config.cpp +++ b/ydb/core/cms/console/yaml_config/yaml_config.cpp @@ -1,8 +1,10 @@ #include "yaml_config.h" #include "yaml_config_impl.h" -#include <library/cpp/protobuf/json/json2proto.h> #include <ydb/core/base/appdata.h> +#include <ydb/library/yaml_config/yaml_config_parser.h> + +#include <library/cpp/protobuf/json/json2proto.h> template <> struct THash<NYamlConfig::TLabel> { @@ -36,6 +38,7 @@ inline const TStringBuf inheritMapTag{"!inherit"}; inline const TStringBuf inheritSeqTag{"!inherit:"}; inline const TStringBuf inheritMapInSeqTag{"!inherit"}; inline const TStringBuf removeTag{"!remove"}; +inline const TStringBuf appendTag{"!append"}; TString GetKey(const NFyaml::TNodeRef& node, TString key) { auto map = node.Map(); @@ -219,6 +222,22 @@ bool IsSeqInherit(const NFyaml::TNodeRef& node) { return false; } +void Append(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from); + +bool IsSeqAppend(const NFyaml::TNodeRef& node) { + if (auto tag = node.Tag(); tag) { + switch (node.Type()) { + case NFyaml::ENodeType::Mapping: + return false; + case NFyaml::ENodeType::Sequence: + return *tag == appendTag; + case NFyaml::ENodeType::Scalar: + return false; + } + } + return false; +} + bool IsRemove(const NFyaml::TNodeRef& node) { if (auto tag = node.Tag(); tag) { return *tag == removeTag; @@ -234,6 +253,8 @@ void Inherit(NFyaml::TMapping& toMap, const NFyaml::TMapping& fromMap) { if (IsMapInherit(fromNode)) { Apply(toNode, fromNode); + } else if (IsSeqAppend(fromNode)) { + Append(toNode, fromNode); } else { toMap.Remove(toEntry.Key()); toMap.Append(it->Key().Copy().Ref(), it->Value().Copy().Ref()); @@ -257,6 +278,8 @@ void Inherit(NFyaml::TSequence& toSeq, const NFyaml::TSequence& fromSeq, const T if (nodes.contains(fromKey)) { if (IsSeqInherit(*it)) { Apply(nodes[fromKey], *it); + } else if (IsSeqAppend(*it)) { + Append(nodes[fromKey], *it); } else if (IsRemove(*it)) { toSeq.Remove(nodes[fromKey]); nodes.erase(fromKey); @@ -274,6 +297,21 @@ void Inherit(NFyaml::TSequence& toSeq, const NFyaml::TSequence& fromSeq, const T } } +void Append(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from) { + Y_ENSURE_EX(to, TYamlConfigEx() << "Appending to empty value: " + << to.Path() << " <- " << from.Path()); + Y_ENSURE_EX(to.Type() == NFyaml::ENodeType::Sequence && from.Type() == NFyaml::ENodeType::Sequence, TYamlConfigEx() << "Appending to wrong type" + << to.Path() << " <- " << from.Path()); + + auto fromSeq = from.Sequence(); + auto toSeq = to.Sequence(); + + for (auto it = fromSeq.begin(); it != fromSeq.end(); ++it) { + auto newNode = it->Copy(); + toSeq.Append(newNode.Ref()); + } +} + void Apply(NFyaml::TNodeRef& to, const NFyaml::TNodeRef& from) { Y_ENSURE_EX(to, TYamlConfigEx() << "Overriding empty value: " << to.Path() << " <- " << from.Path()); @@ -401,6 +439,8 @@ NKikimrConfig::TAppConfig YamlToProto(const NFyaml::TNodeRef& node, bool allowUn NJson::ReadJsonTree(resolvedJsonConfig, &json); + NKikimr::NYaml::TransformConfig(json, true); + NKikimrConfig::TAppConfig yamlProtoConfig; NProtobufJson::TJson2ProtoConfig c; @@ -552,7 +592,7 @@ size_t Hash(const TResolvedConfig& config) void ValidateVolatileConfig(NFyaml::TDocument& doc) { auto root = doc.Root(); - auto seq = root.Sequence(); + auto seq = root.Map().at("selector_config").Sequence(); if (seq.size() == 0) { ythrow yexception() << "Empty volatile config"; } @@ -588,13 +628,228 @@ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatil } } +void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig) { + auto configRoot = config.Root(); + + auto seq = volatileConfig.Sequence(); + auto selectors = configRoot.Map().at("selector_config").Sequence(); + for (auto& elem : seq) { + auto node = elem.Copy(config); + selectors.Append(node.Ref()); + } +} + ui64 GetVersion(const TString& config) { - auto parser = NFyaml::TParser::Create(config); - auto header = parser.NextDocument(); - auto str = header->Root().Map().at("version").Scalar(); - ui64 version = 0; - TryFromString<ui64>(str, version); - return version; + auto metadata = GetMetadata(config); + return metadata.Version.value_or(0); +} + +/** + * Config used to convert protobuf from/to json + * changes how names are translated e.g. PDiskInfo -> pdisk_info instead of p_disk_info + */ +NProtobufJson::TJson2ProtoConfig GetJsonToProtoConfig() { + NProtobufJson::TJson2ProtoConfig config; + config.SetFieldNameMode(NProtobufJson::TJson2ProtoConfig::FieldNameSnakeCaseDense); + config.SetEnumValueMode(NProtobufJson::TJson2ProtoConfig::EnumCaseInsensetive); + config.CastRobust = true; + config.MapAsObject = true; + config.AllowUnknownFields = false; + return config; +} + +void ResolveAndParseYamlConfig( + const TString& yamlConfig, + const TMap<ui64, TString>& volatileYamlConfigs, + const TMap<TString, TString>& labels, + NKikimrConfig::TAppConfig& appConfig, + TString* resolvedYamlConfig, + TString* resolvedJsonConfig) { + + auto tree = NFyaml::TDocument::Parse(yamlConfig); + + for (auto& [_, config] : volatileYamlConfigs) { + auto d = NFyaml::TDocument::Parse(config); + NYamlConfig::AppendVolatileConfigs(tree, d); + } + + TSet<NYamlConfig::TNamedLabel> namedLabels; + for (auto& [name, label] : labels) { + namedLabels.insert(NYamlConfig::TNamedLabel{name, label}); + } + + auto config = NYamlConfig::Resolve(tree, namedLabels); + + if (resolvedYamlConfig) { + TStringStream resolvedYamlConfigStream; + resolvedYamlConfigStream << config.second; + *resolvedYamlConfig = resolvedYamlConfigStream.Str(); + } + + TStringStream resolvedJsonConfigStream; + resolvedJsonConfigStream << NFyaml::TJsonEmitter(config.second); + + if (resolvedJsonConfig) { + *resolvedJsonConfig = resolvedJsonConfigStream.Str(); + } + + NJson::TJsonValue json; + Y_VERIFY(NJson::ReadJsonTree(resolvedJsonConfigStream.Str(), &json), "Got invalid config from Console"); + + NKikimr::NYaml::TransformConfig(json, true); + + NProtobufJson::MergeJson2Proto(json, appConfig, NYamlConfig::GetJsonToProtoConfig()); +} + +void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig::TAppConfig& to) { + if (from.HasNameserviceConfig()) { + to.MutableNameserviceConfig()->CopyFrom(from.GetNameserviceConfig()); + } + + if (from.HasNetClassifierDistributableConfig()) { + to.MutableNetClassifierDistributableConfig()->CopyFrom(from.GetNetClassifierDistributableConfig()); + } + + if (from.NamedConfigsSize()) { + to.MutableNamedConfigs()->CopyFrom(from.GetNamedConfigs()); + } +} + +TMetadata GetMetadata(const TString& config) { + if (config.empty()) { + return {}; + } + + auto doc = NFyaml::TDocument::Parse(config); + + if (auto node = doc.Root().Map()["metadata"]; node) { + auto versionNode = node.Map()["version"]; + auto clusterNode = node.Map()["cluster"]; + return TMetadata{ + .Version = versionNode ? std::optional{FromString<ui64>(versionNode.Scalar())} : std::nullopt, + .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, + }; + } + + return {}; +} + +TVolatileMetadata GetVolatileMetadata(const TString& config) { + if (config.empty()) { + return {}; + } + + auto doc = NFyaml::TDocument::Parse(config); + + if (auto node = doc.Root().Map().at("metadata"); node) { + auto versionNode = node.Map().at("version"); + auto clusterNode = node.Map().at("cluster"); + auto idNode = node.Map().at("id"); + return TVolatileMetadata{ + .Version = versionNode ? std::make_optional(FromString<ui64>(versionNode.Scalar())) : std::nullopt, + .Cluster = clusterNode ? std::make_optional(clusterNode.Scalar()) : std::nullopt, + .Id = idNode ? std::make_optional(FromString<ui64>(idNode.Scalar())) : std::nullopt, + }; + } + + return {}; +} + +TString ReplaceMetadata(const TString& config, const std::function<void(TStringStream&)>& serializeMetadata) { + TStringStream sstr; + auto doc = NFyaml::TDocument::Parse(config); + if (doc.Root().Style() == NFyaml::ENodeStyle::Flow) { + if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { + doc.Root().Map().Remove(pair); + } + serializeMetadata(sstr); + sstr << "\n" << doc; + } else { + if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { + auto begin = pair.Key().BeginMark().InputPos; + auto end = pair.Value().EndMark().InputPos; + sstr << config.substr(0, begin); + serializeMetadata(sstr); + if (end < config.length() && config[end] == ':') { + end = end + 1; + } + sstr << config.substr(end, TString::npos); + } else { + if (doc.HasExplicitDocumentStart()) { + auto docStart = doc.BeginMark().InputPos + 4; + sstr << config.substr(0, docStart); + serializeMetadata(sstr); + sstr << "\n" << config.substr(docStart, TString::npos); + } else { + serializeMetadata(sstr); + sstr << "\n" << config; + } + } + } + return sstr.Str(); + +} + +TString ReplaceMetadata(const TString& config, const TMetadata& metadata) { + auto serializeMetadata = [&](TStringStream& sstr) { + sstr + << "metadata:" + << "\n kind: MainConfig" + << "\n cluster: \"" << *metadata.Cluster << "\"" + << "\n version: " << *metadata.Version; + }; + return ReplaceMetadata(config, serializeMetadata); +} + +TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata) { + auto serializeMetadata = [&](TStringStream& sstr) { + sstr + << "metadata:" + << "\n kind: VolatileConfig" + << "\n cluster: \"" << *metadata.Cluster << "\"" + << "\n version: " << *metadata.Version + << "\n id: " << *metadata.Id; + }; + return ReplaceMetadata(config, serializeMetadata); +} + +bool IsConfigKindEquals(const TString& config, const TString& kind) { + try { + auto doc = NFyaml::TDocument::Parse(config); + return doc.Root().Map().at("metadata").Map().at("kind").Scalar() == kind; + } catch (yexception& e) { + return false; + } +} + +bool IsVolatileConfig(const TString& config) { + return IsConfigKindEquals(config, "VolatileConfig"); +} + +bool IsMainConfig(const TString& config) { + return IsConfigKindEquals(config, "MainConfig"); +} + +TString StripMetadata(const TString& config) { + auto doc = NFyaml::TDocument::Parse(config); + + TStringStream sstr; + if (auto pair = doc.Root().Map().pair_at_opt("metadata"); pair) { + auto begin = pair.Key().BeginMark().InputPos; + sstr << config.substr(0, begin); + auto end = pair.Value().EndMark().InputPos; + sstr << config.substr(end, TString::npos); + } else { + if (doc.HasExplicitDocumentStart()) { + auto docStart = doc.BeginMark().InputPos + 4; + sstr << config.substr(0, docStart); + sstr << "\n" << config.substr(docStart, TString::npos); + } else { + sstr << config; + } + } + + return sstr.Str(); } } // namespace NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/yaml_config.h b/ydb/core/cms/console/yaml_config/yaml_config.h index 08f0c63fad..790041adb8 100644 --- a/ydb/core/cms/console/yaml_config/yaml_config.h +++ b/ydb/core/cms/console/yaml_config/yaml_config.h @@ -164,12 +164,89 @@ void ValidateVolatileConfig(NFyaml::TDocument& doc); /** * Appends volatile configs to the end of selectors list + * **Important**: Document should be a list with selectors */ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatileConfig); /** + * Appends volatile configs to the end of selectors list + * **Important**: Node should be a list with selectors + */ +void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig); + +/** * Parses config version */ ui64 GetVersion(const TString& config); +/** + * Resolves config for given labels and stores result to appConfig + * Stores intermediate resolve data in resolvedYamlConfig and resolvedJsonConfig if given + */ +void ResolveAndParseYamlConfig( + const TString& yamlConfig, + const TMap<ui64, TString>& volatileYamlConfigs, + const TMap<TString, TString>& labels, + NKikimrConfig::TAppConfig& appConfig, + TString* resolvedYamlConfig = nullptr, + TString* resolvedJsonConfig = nullptr); + +/** + * Replaces kinds not managed by yaml config (e.g. NetClassifierConfig) from config 'from' in config 'to' + * if corresponding configs are presenet in 'from' + */ +void ReplaceUnmanagedKinds(const NKikimrConfig::TAppConfig& from, NKikimrConfig::TAppConfig& to); + +/** + * Represents config metadata + */ +struct TMetadata { + std::optional<ui64> Version; + std::optional<TString> Cluster; +}; + +/** + * Parses config metadata + */ +TMetadata GetMetadata(const TString& config); + +/** + * Represents volatile config metadata + */ +struct TVolatileMetadata { + std::optional<ui64> Version; + std::optional<TString> Cluster; + std::optional<ui64> Id; +}; + +/** + * Parses volatile config metadata + */ +TVolatileMetadata GetVolatileMetadata(const TString& config); + +/** + * Replaces metadata in config + */ +TString ReplaceMetadata(const TString& config, const TMetadata& metadata); + +/** + * Replaces volatile metadata in config + */ +TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata); + +/** + * Checks whether string is volatile config or not + */ +bool IsVolatileConfig(const TString& config); + +/** + * Checks whether string is main config or not + */ +bool IsMainConfig(const TString& config); + +/** + * Strips metadata from config + */ +TString StripMetadata(const TString& config); + } // namespace NYamlConfig diff --git a/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp b/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp index fd558e60eb..8885465758 100644 --- a/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp +++ b/ydb/core/cms/console/yaml_config/yaml_config_ut.cpp @@ -742,36 +742,62 @@ config: allowed_labels: tenant: type: string -selector_config: [ - { - description: test 4, - selector: { - tenant: /dev_global - }, - config: { - actor_system_config: {}, - cms_config: { - sentinel_config: { - enable: false - } - } - } - }, - { - description: test 5, - selector: { - canary: true - }, - config: { - actor_system_config: {}, - cms_config: { - sentinel_config: { - enable: true - } - } - } - } - ] +selector_config: +- description: test 4 + selector: + tenant: /dev_global + config: + actor_system_config: {} + cms_config: + sentinel_config: + enable: false +- description: test 5 + selector: + canary: true + config: + actor_system_config: {} + cms_config: + sentinel_config: + enable: true +)"; + +const char *UnresolvedSimpleConfigAppend = R"(--- +cluster: test +version: 12.1 +config: + num: + - 0 +allowed_labels: + tenant: + type: string + +selector_config: +- description: 1 + selector: + tenant: abc + config: !inherit + num: !append + - 0 +- description: 2 + selector: + tenant: "" + config: !inherit + num: !append + - 1 +)"; + +const char *ResolvedSimpleConfigAppendAbc = R"(--- +config: + num: + - 0 + - 0 +)"; + +const char *ResolvedSimpleConfigAppendEmpty = R"(--- +config: + num: + - 0 + - 1 )"; using EType = NYamlConfig::TLabel::EType; @@ -1364,6 +1390,31 @@ Y_UNIT_TEST_SUITE(YamlConfig) { UNIT_ASSERT_UNEQUAL(secondIt, resolved.Configs.end()); UNIT_ASSERT(secondIt->second.second.DeepEqual(abc.Root().Map()["config"])); } + + { + auto doc = NFyaml::TDocument::Parse(UnresolvedSimpleConfigAppend); + auto resolved = NYamlConfig::ResolveAll(doc); + UNIT_ASSERT_VALUES_EQUAL(resolved.Labels, expectedLabels); + + auto empty = NFyaml::TDocument::Parse(ResolvedSimpleConfigAppendEmpty); + auto abc = NFyaml::TDocument::Parse(ResolvedSimpleConfigAppendAbc); + + TSet<TVector<NYamlConfig::TLabel>> first = { + {NYamlConfig::TLabel{EType::Empty, ""}}, + }; + + auto firstIt = resolved.Configs.find(first); + UNIT_ASSERT_UNEQUAL(firstIt, resolved.Configs.end()); + UNIT_ASSERT(firstIt->second.second.DeepEqual(empty.Root().Map()["config"])); + + TSet<TVector<NYamlConfig::TLabel>> second = { + {NYamlConfig::TLabel{EType::Common, "abc"}}, + }; + + auto secondIt = resolved.Configs.find(second); + UNIT_ASSERT_UNEQUAL(secondIt, resolved.Configs.end()); + UNIT_ASSERT(secondIt->second.second.DeepEqual(abc.Root().Map()["config"])); + } } Y_UNIT_TEST(MaterializeAllConfigs) { @@ -1415,4 +1466,303 @@ Y_UNIT_TEST_SUITE(YamlConfig) { TStringStream stream; stream << cfg; } + + Y_UNIT_TEST(GetMetadata) { + { + TString str = R"( +metadata: + version: 10 + cluster: foo +)"; + auto metadata = NYamlConfig::GetMetadata(str); + UNIT_ASSERT_VALUES_EQUAL(*metadata.Version, 10); + UNIT_ASSERT_VALUES_EQUAL(*metadata.Cluster, "foo"); + } + + { + TString str = R"( +metadata: + version: 10 +)"; + auto metadata = NYamlConfig::GetMetadata(str); + UNIT_ASSERT_VALUES_EQUAL(*metadata.Version, 10); + UNIT_ASSERT(!metadata.Cluster); + } + + { + TString str = R"( +metadata: + cluster: foo +)"; + auto metadata = NYamlConfig::GetMetadata(str); + UNIT_ASSERT(!metadata.Version); + UNIT_ASSERT_VALUES_EQUAL(*metadata.Cluster, "foo"); + } + + { + TString str = R"( +metadata: {} +)"; + auto metadata = NYamlConfig::GetMetadata(str); + UNIT_ASSERT(!metadata.Version); + UNIT_ASSERT(!metadata.Cluster); + } + + { + TString str = "foo: bar"; + auto metadata = NYamlConfig::GetMetadata(str); + UNIT_ASSERT(!metadata.Version); + UNIT_ASSERT(!metadata.Cluster); + } + } + + Y_UNIT_TEST(ReplaceMetadata) { + NYamlConfig::TMetadata metadata; + metadata.Version = 1; + metadata.Cluster = "test"; + + { + TString str = R"( +# comment1 +{value: 1, array: [{test: "1"}], obj: {value: 2}} # comment2 +# comment3 +)"; + + TString exp = R"(metadata: + kind: MainConfig + cluster: "test" + version: 1 +value: 1 +array: +- test: "1" +obj: + value: 2 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"( +# comment1 +{value: 1, metadata: {version: 1, cluster: "test"}, array: [{test: "1"}], obj: {value: 2}} # comment2 +# comment3 +)"; + + TString exp = R"(metadata: + kind: MainConfig + cluster: "test" + version: 1 +value: 1 +array: +- test: "1" +obj: + value: 2 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"(# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"(metadata: + kind: MainConfig + cluster: "test" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"(metadata: {version: 0, cluster: tes} +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"(metadata: + kind: MainConfig + cluster: "test" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"(metadata: + version: 0 + cluster: tes +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"(metadata: + kind: MainConfig + cluster: "test" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"(metadata: + version: 0 + cool: {foo: bar} + cluster: tes +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"(metadata: + kind: MainConfig + cluster: "test" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"( + +--- +metadata: + cluster: tes + version: 0 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"( + +--- +metadata: + kind: MainConfig + cluster: "test" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + { + TString str = R"( +--- +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"( +--- +metadata: + kind: MainConfig + cluster: "test" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + + metadata.Cluster = ""; + + { + TString str = R"( +--- +metadata: + version: 1 + cluster: +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString exp = R"( +--- +metadata: + kind: MainConfig + cluster: "" + version: 1 +# comment1 +value: 1 +array: [{test: "1"}] +obj: {value: 2} # comment2 +# comment3 +)"; + + TString res = NYamlConfig::ReplaceMetadata(str, metadata); + + UNIT_ASSERT_VALUES_EQUAL(res, exp); + } + } } diff --git a/ydb/core/cms/json_proxy.h b/ydb/core/cms/json_proxy.h index c833f20b35..510af4f661 100644 --- a/ydb/core/cms/json_proxy.h +++ b/ydb/core/cms/json_proxy.h @@ -18,6 +18,8 @@ #include <library/cpp/protobuf/json/json2proto.h> #include <library/cpp/protobuf/json/proto2json.h> +#include <library/cpp/json/json_writer.h> + #include <iostream> namespace NKikimr::NCms { @@ -91,6 +93,9 @@ protected: { switch (ev->GetTypeRewrite()) { HFunc(TResponseEvent, Handle); + HFunc(NConsole::TEvConsole::TEvUnauthorized, HandleError); + HFunc(NConsole::TEvConsole::TEvDisabled, HandleError); + HFunc(NConsole::TEvConsole::TEvGenericError, HandleError); CFunc(TEvents::TSystem::Wakeup, Timeout); CFunc(TEvTabletPipe::TEvClientDestroyed::EventType, Disconnect); HFunc(TEvTabletPipe::TEvClientConnected, Handle); @@ -139,9 +144,37 @@ protected: ReplyAndDie(ev->Get()->Record, ctx); } - void SetTempError(NKikimrCms::TStatus &status, - const TString &error) - { + void HandleError(NConsole::TEvConsole::TEvUnauthorized::TPtr &, const TActorContext &ctx) { + ReplyAndDieImpl(TString(NMonitoring::HTTPUNAUTHORIZED), ctx); + } + + void HandleError(NConsole::TEvConsole::TEvDisabled::TPtr &, const TActorContext &ctx) { + ReplyAndDieImpl(TString("HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n{\"code\":400, \"message\":\"Feature is disabled\"}\r\n"), ctx); + } + + void HandleError(NConsole::TEvConsole::TEvGenericError::TPtr &ev, const TActorContext &ctx) { + TStringStream issues; + for (auto& issue : ev->Get()->Record.GetIssues()) { + issues << issue.ShortDebugString() + ", "; + } + + TString res; + TStringOutput ss(res); + + NJson::TJsonWriter writer(&ss, true); + + writer.OpenMap(); + writer.Write("code", (ui64)ev->Get()->Record.GetYdbStatus()); + writer.Write("issues", issues.Str()); + writer.CloseMap(); + + writer.Flush(); + ss.Flush(); + + ReplyAndDieImpl(TString("HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n") + res + "\r\n", ctx); + } + + void SetTempError(NKikimrCms::TStatus &status, const TString &error) { status.SetCode(NKikimrCms::TStatus::ERROR_TEMP); status.SetReason(error); } diff --git a/ydb/core/cms/ui/config_dispatcher.css b/ydb/core/cms/ui/config_dispatcher.css new file mode 100644 index 0000000000..fe0aa308cd --- /dev/null +++ b/ydb/core/cms/ui/config_dispatcher.css @@ -0,0 +1,6 @@ +.CodeMirror { + border: 1px solid #eee; + height: auto; + display: flex; + width: 1110px; +} diff --git a/ydb/core/cms/ui/configs_dispatcher_main.js b/ydb/core/cms/ui/configs_dispatcher_main.js new file mode 100644 index 0000000000..7af3993f5f --- /dev/null +++ b/ydb/core/cms/ui/configs_dispatcher_main.js @@ -0,0 +1,120 @@ +var createEditor; + +var codeMirror; +var codeMirrorResolved; + +function replaceWithEditor(selector) { + var container = $(selector); + var value = container.text(); + container.text(""); + var editor = createEditor(container.get(0), true, 1068) + editor.setValue(value); + return editor; +} + +function main() { + $("#nodePicker").fuzzyComplete(nodeNames); + $('#nodePicker').on('keyup blur', function() { + if (window.location.pathname === "/actors/configs_dispatcher") { + $('#nodesGo').attr("href", window.location.protocol + "//" + $(this).parent().find("select").val() + ":8765/actors/configs_dispatcher"); + } else { + $('#nodesGo').attr("href", "/" + $(this).parent().find("select").val() + ":8765/actors/configs_dispatcher"); + } + }); + + codeMirror = replaceWithEditor("#yaml-config-item"); + codeMirror.trigger('fold', 'editor.foldLevel2'); + + $("#fold-yaml-config").click(function() { + codeMirror.trigger('fold', 'editor.foldLevel2'); + }); + + $("#unfold-yaml-config").click(function() { + codeMirror.trigger('fold', 'editor.unfoldAll'); + }); + + $("#copy-yaml-config").click(function() { + copyToClipboard(codeMirror.getValue()); + }); + + codeMirrorResolved = replaceWithEditor("#resolved-yaml-config-item"); + codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); + + $("#fold-resolved-yaml-config").click(function() { + codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); + }); + + $("#unfold-resolved-yaml-config").click(function() { + codeMirrorResolved.trigger('fold', 'editor.unfoldAll'); + }); + + $("#copy-resolved-yaml-config").click(function() { + copyToClipboard(codeMirrorResolved.getValue()); + }); + + $("#host-ref").text("YDB Developer UI - " + window.location.hostname); + + $(".yaml-config-item").each(function() { + let editor = replaceWithEditor(this); + + $(this).parent().find('.fold-yaml-config').click(function() { + editor.trigger('fold', 'editor.foldLevel2'); + }); + + $(this).parent().find('.unfold-yaml-config').click(function() { + editor.trigger('fold', 'editor.unfoldAll'); + }); + + $(this).parent().find('.copy-yaml-config').click(function() { + copyToClipboard(editor.getValue()); + }); + }); +} + +let run = () => { + require.config({ + paths: { vs: "https://cdn.jsdelivr.net/npm/monaco-editor@0.27.0/min/vs" } + }); + + require(["vs/editor/editor.main"], function () { + createEditor = (container, readOnly, width) => { + var editor; + container.style.border = '1px solid #eee'; + container.style.borderRadius = '8px'; + container.style.overflow = 'hidden'; + editor = monaco.editor.create(container, { + language: "yaml", + scrollBeyondLastLine: false, + wrappingStrategy: 'advanced', + minimap: { + enabled: false, + }, + overviewRulerLanes: 0, + automaticLayout: true, + readOnly: readOnly, + scrollbar: { + alwaysConsumeMouseWheel: false, + }, + }); + let ignoreEvent = false; + const updateHeight = () => { + const contentHeight = editor.getContentHeight(); + container.style.width = `100%`; + container.style.height = `${contentHeight + 2}px`; + try { + ignoreEvent = true; + editor.layout({ width, height: contentHeight }); + } finally { + ignoreEvent = false; + } + }; + editor.onDidContentSizeChange(updateHeight); + updateHeight(); + return editor; + } + + $(document).ready(main); + }); +}; + +run(); diff --git a/ydb/core/cms/ui/yaml_config.js b/ydb/core/cms/ui/yaml_config.js index 9377021217..b86c7a8bb9 100644 --- a/ydb/core/cms/ui/yaml_config.js +++ b/ydb/core/cms/ui/yaml_config.js @@ -168,16 +168,16 @@ class YamlConfigState { loadYaml() { clearTimeout(this.loadYamlTimeout); - $.get(this.url).done(this.onYamlLoaded.bind(this)).fail(this.onYamlLoaded.bind(this)); + $.get(this.url).done(this.onYamlLoaded.bind(this, true)).fail(this.onYamlLoaded.bind(this, false)); } - onYamlLoaded(data) { - if (data?.Response?.operation?.status === "SUCCESS") { - this.cluster = data.Response.cluster; - $('#yaml-cluster').text(data.Response.cluster); + onYamlLoaded(success, data) { + if (success) { + this.cluster = data.Response.identity.cluster; + $('#yaml-cluster').text(data.Response.identity.cluster); - this.version = data.Response.version; - $('#yaml-version').text(data.Response.version); + this.version = data.Response.identity.version; + $('#yaml-version').text(data.Response.identity.version); if (this.config !== data.Response.config) { this.codeMirror.setValue((data.Response.config !== undefined) ? data.Response.config : ""); @@ -246,11 +246,11 @@ class YamlConfigState { this.loadYamlTimeout = setTimeout(this.loadYaml.bind(this), this.fetchInterval); } - onVolatileConfigChanged(data) { - if (data?.Response?.operation?.status === "SUCCESS") { + onVolatileConfigChanged(success, data) { + if (success) { this.loadYaml(); } else { - showToast("Error", "Invalid volatile config\n" + data.Response.operation.issues[0].message, 15000); + showToast("Error", "Invalid volatile config\n" + data.responseJSON.issues, 15000); } } @@ -265,8 +265,8 @@ class YamlConfigState { }; $.post(this.applyUrl, JSON.stringify(cmd)) - .done(this.onVolatileConfigChanged.bind(this)) - .fail(this.onVolatileConfigChanged.bind(this)); + .done(this.onVolatileConfigChanged.bind(this, true)) + .fail(this.onVolatileConfigChanged.bind(this, false)); this.volatileCodeMirror.setValue(""); @@ -284,8 +284,8 @@ class YamlConfigState { }; $.post(this.removeVolatileUrl, JSON.stringify(cmd)) - .done(this.onVolatileConfigChanged.bind(this)) - .fail(this.onVolatileConfigChanged.bind(this)); + .done(this.onVolatileConfigChanged.bind(this, true)) + .fail(this.onVolatileConfigChanged.bind(this, false)); } removeVolatileConfig(id) { @@ -298,15 +298,15 @@ class YamlConfigState { }; $.post(this.removeVolatileUrl, JSON.stringify(cmd)) - .done(this.onVolatileConfigChanged.bind(this)) - .fail(this.onVolatileConfigChanged.bind(this)); + .done(this.onVolatileConfigChanged.bind(this, true)) + .fail(this.onVolatileConfigChanged.bind(this, false)); } - onResolved(data) { - if (data?.Response?.operation?.status === "SUCCESS") { + onResolved(success, data) { + if (success) { this.resolvedCodeMirror.setValue(data.Response.config); } else { - showToast("Error", "Config resolution error\n" + data.Response.operation.issues[0].message, 15000); + showToast("Error", "Config resolution error\n" + data.responseJSON.issues, 15000); } } @@ -320,8 +320,8 @@ class YamlConfigState { }; $.post(this.resolveAllUrl, JSON.stringify(cmd)) - .done(this.onResolved.bind(this)) - .fail(this.onResolved.bind(this)); + .done(this.onResolved.bind(this, true)) + .fail(this.onResolved.bind(this, false)); } resolveForLabels() { @@ -342,8 +342,8 @@ class YamlConfigState { }; $.post(this.resolveUrl, JSON.stringify(cmd)) - .done(this.onResolved.bind(this)) - .fail(this.onResolved.bind(this)); + .done(this.onResolved.bind(this, true)) + .fail(this.onResolved.bind(this, false)); } initTab() { diff --git a/ydb/core/docapi/CMakeLists.darwin.txt b/ydb/core/docapi/CMakeLists.darwin.txt new file mode 100644 index 0000000000..43f062ce15 --- /dev/null +++ b/ydb/core/docapi/CMakeLists.darwin.txt @@ -0,0 +1,20 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-docapi) +target_compile_options(ydb-core-docapi PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-docapi PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(ydb-core-docapi PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/docapi/traits.cpp +) diff --git a/ydb/core/docapi/CMakeLists.linux-aarch64.txt b/ydb/core/docapi/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..abffc1e9aa --- /dev/null +++ b/ydb/core/docapi/CMakeLists.linux-aarch64.txt @@ -0,0 +1,21 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-docapi) +target_compile_options(ydb-core-docapi PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-docapi PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(ydb-core-docapi PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/docapi/traits.cpp +) diff --git a/ydb/core/docapi/CMakeLists.linux.txt b/ydb/core/docapi/CMakeLists.linux.txt new file mode 100644 index 0000000000..abffc1e9aa --- /dev/null +++ b/ydb/core/docapi/CMakeLists.linux.txt @@ -0,0 +1,21 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-docapi) +target_compile_options(ydb-core-docapi PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-docapi PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(ydb-core-docapi PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/docapi/traits.cpp +) diff --git a/ydb/services/console/ut/CMakeLists.txt b/ydb/core/docapi/CMakeLists.txt index 3e0811fb22..3e0811fb22 100644 --- a/ydb/services/console/ut/CMakeLists.txt +++ b/ydb/core/docapi/CMakeLists.txt diff --git a/ydb/core/docapi/traits.cpp b/ydb/core/docapi/traits.cpp new file mode 100644 index 0000000000..38a6e8cebf --- /dev/null +++ b/ydb/core/docapi/traits.cpp @@ -0,0 +1,8 @@ +#include "traits.h" + +namespace NKikimr::NDocApi { + +const TStringBuf RequestType = "_document_api_request"; +const TStringBuf VersionAttribute = "__document_api_version"; + +} diff --git a/ydb/core/docapi/traits.h b/ydb/core/docapi/traits.h new file mode 100644 index 0000000000..843b7b86e7 --- /dev/null +++ b/ydb/core/docapi/traits.h @@ -0,0 +1,10 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NKikimr::NDocApi { + +extern const TStringBuf RequestType; +extern const TStringBuf VersionAttribute; + +} diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.darwin.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.darwin.txt index bab2c0e260..a9bc628c5a 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.darwin.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.darwin.txt @@ -23,6 +23,7 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib + cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt index 74ce26edb5..4abec26df5 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.linux-aarch64.txt @@ -24,6 +24,7 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib + cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/CMakeLists.linux.txt b/ydb/core/driver_lib/cli_utils/CMakeLists.linux.txt index 74ce26edb5..4abec26df5 100644 --- a/ydb/core/driver_lib/cli_utils/CMakeLists.linux.txt +++ b/ydb/core/driver_lib/cli_utils/CMakeLists.linux.txt @@ -24,6 +24,7 @@ target_link_libraries(cli_utils PUBLIC core-blobstorage-pdisk core-client-minikql_compile core-client-scheme_cache_lib + cms-console-yaml_config cli_base ydb-core-engine ydb-core-erasure diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp index 1091cad327..3e75e90895 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp @@ -1,7 +1,9 @@ -#include <util/string/type.h> #include "cli.h" #include "cli_cmds.h" +#include <ydb/core/cms/console/yaml_config/console_dumper.h> + +#include <util/string/type.h> #include <util/string/split.h> #include <util/system/fs.h> @@ -313,12 +315,44 @@ public: } }; +class TClientCommandConsoleConfigsDumpYaml : public TConsoleClientCommand { +public: + TClientCommandConsoleConfigsDumpYaml() + : TConsoleClientCommand("dump-yaml", {}, "Dump config in yaml format") + { + } + + virtual void Config(TConfig& config) override { + TConsoleClientCommand::Config(config); + config.SetFreeArgsNum(0); + } + + virtual void Parse(TConfig& config) override { + TConsoleClientCommand::Parse(config); + Request.MutableGetConfigItemsRequest(); + } + + virtual void PrintResponse(const NKikimrClient::TConsoleResponse &response) override + { + if (response.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { + Cout << "ERROR: " << response.GetStatus().GetCode() + << " (" << response.GetStatus().GetReason() << ")" << Endl; + return; + } + + auto &items = response.GetGetConfigItemsResponse().GetConfigItems(); + + Cout << NYamlConfig::DumpConsoleConfigs(items); + } +}; + class TClientCommandConsoleConfigs : public TClientCommandTree { public: TClientCommandConsoleConfigs() : TClientCommandTree("configs", {}, "") { AddCommand(std::make_unique<TClientCommandConsoleConfigsLoad>()); + AddCommand(std::make_unique<TClientCommandConsoleConfigsDumpYaml>()); AddCommand(std::make_unique<TClientCommandConsoleConfigsUpdate>()); } }; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 298b5d354f..5141c62ed6 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -2,6 +2,7 @@ #include "cli_cmds.h" #include <ydb/core/base/location.h> #include <ydb/core/base/path.h> +#include <ydb/core/cms/console/yaml_config/yaml_config.h> #include <ydb/core/driver_lib/run/run.h> #include <ydb/library/yaml_config/yaml_config_parser.h> #include <ydb/public/lib/deprecated/kicli/kicli.h> @@ -317,6 +318,19 @@ protected: return 0; } + void AddLabelToAppConfig(const TString& name, const TString& value) { + for (auto &label : *RunConfig.AppConfig.MutableLabels()) { + if (label.GetName() == name) { + label.SetValue(value); + return; + } + } + + auto *label = RunConfig.AppConfig.AddLabels(); + label->SetName(name); + label->SetValue(value); + } + virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); @@ -380,7 +394,38 @@ protected: ythrow yexception() << "wrong '--node-kind' value '" << NodeKind << "', only '" << NODE_KIND_YDB << "' or '" << NODE_KIND_YQ << "' is allowed"; } - MaybeRegisterAndLoadConfigs(); + RunConfig.Labels["node_id"] = ToString(NodeId); + RunConfig.Labels["node_host"] = FQDNHostName(); + RunConfig.Labels["tenant"] = TenantName; + RunConfig.Labels["node_type"] = NodeType; + // will be replaced with proper version info + RunConfig.Labels["branch"] = GetBranch(); + RunConfig.Labels["rev"] = GetProgramCommitId(); + RunConfig.Labels["dynamic"] = ToString(NodeBrokerAddresses.empty() ? "false" : "true"); + + for (const auto& [name, value] : RunConfig.Labels) { + auto *label = RunConfig.AppConfig.AddLabels(); + label->SetName(name); + label->SetValue(value); + } + + // static node + if (NodeBrokerAddresses.empty() && !NodeBrokerPort) { + if (!NodeId) { + ythrow yexception() << "Either --node [NUM|'static'] or --node-broker[-port] should be specified"; + } + + if (!HierarchicalCfg && RunConfig.PathToConfigCacheFile) + LoadCachedConfigsForStaticNode(); + } else { + RegisterDynamicNode(); + + RunConfig.Labels["node_id"] = ToString(RunConfig.NodeId); + AddLabelToAppConfig("node_id", RunConfig.Labels["node_id"]); + + if (!HierarchicalCfg && !IgnoreCmsConfigs) + LoadConfigForDynamicNode(); + } LoadYamlConfig(); @@ -688,20 +733,13 @@ protected: messageBusConfig->SetTracePath(TracePath); } - RunConfig.Labels["node_id"] = ToString(NodeId); - RunConfig.Labels["node_host"] = FQDNHostName(); - RunConfig.Labels["tenant"] = RunConfig.TenantName; - // will be replaced with proper version info - RunConfig.Labels["branch"] = GetBranch(); - RunConfig.Labels["rev"] = ToString(GetProgramSvnRevision()); - - for (const auto& [name, value] : RunConfig.Labels) { - auto *label = RunConfig.AppConfig.AddLabels(); - label->SetName(name); - label->SetValue(value); + if (RunConfig.AppConfig.HasDynamicNameserviceConfig()) { + bool isDynamic = RunConfig.NodeId > RunConfig.AppConfig.GetDynamicNameserviceConfig().GetMaxStaticNodeId(); + RunConfig.Labels["dynamic"] = ToString(isDynamic ? "true" : "false"); + AddLabelToAppConfig("node_id", RunConfig.Labels["node_id"]); } - RunConfig.ClusterName = ClusterName; + RunConfig.ClusterName = RunConfig.AppConfig.GetNameserviceConfig().GetClusterUUID(); } inline bool LoadConfigFromCMS() { @@ -731,7 +769,9 @@ protected: TenantName, NodeType, DeduceNodeDomain(), - AppConfig.GetAuthConfig().GetStaffApiUserToken()); + AppConfig.GetAuthConfig().GetStaffApiUserToken(), + true, + 1); if (result.IsSuccess()) { auto appConfig = result.GetConfig(); @@ -743,7 +783,27 @@ protected: } } - BaseConfig.Swap(&appConfig); + NKikimrConfig::TAppConfig yamlConfig; + + if (result.HasYamlConfig() && !result.GetYamlConfig().empty()) { + NYamlConfig::ResolveAndParseYamlConfig( + result.GetYamlConfig(), + result.GetVolatileYamlConfigs(), + RunConfig.Labels, + yamlConfig); + } + + RunConfig.InitialCmsConfig.CopyFrom(appConfig); + + RunConfig.InitialCmsYamlConfig.CopyFrom(yamlConfig); + NYamlConfig::ReplaceUnmanagedKinds(appConfig, RunConfig.InitialCmsYamlConfig); + + if (yamlConfig.HasYamlConfigEnabled() && yamlConfig.GetYamlConfigEnabled()) { + BaseConfig.Swap(&yamlConfig); + NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), BaseConfig); + } else { + BaseConfig.Swap(&appConfig); + } Cout << "Success." << Endl; @@ -871,24 +931,6 @@ protected: } } - void MaybeRegisterAndLoadConfigs() - { - // static node - if (NodeBrokerAddresses.empty() && !NodeBrokerPort) { - if (!NodeId) { - ythrow yexception() << "Either --node [NUM|'static'] or --node-broker[-port] should be specified"; - } - - if (!HierarchicalCfg && RunConfig.PathToConfigCacheFile) - LoadCachedConfigsForStaticNode(); - return; - } - - RegisterDynamicNode(); - if (!HierarchicalCfg && !IgnoreCmsConfigs) - LoadConfigForDynamicNode(); - } - THolder<NClient::TRegistrationResult> TryToRegisterDynamicNode( const TString &addr, const TString &domainName, @@ -1060,7 +1102,9 @@ protected: TenantName, NodeType, DeduceNodeDomain(), - AppConfig.GetAuthConfig().GetStaffApiUserToken()); + AppConfig.GetAuthConfig().GetStaffApiUserToken(), + true, + 1); if (!result.IsSuccess()) { error = result.GetErrorMessage(); @@ -1070,7 +1114,29 @@ protected: Cout << "Success." << Endl; - auto appConfig = result.GetConfig(); + NKikimrConfig::TAppConfig appConfig; + + NKikimrConfig::TAppConfig yamlConfig; + + if (result.HasYamlConfig() && !result.GetYamlConfig().empty()) { + NYamlConfig::ResolveAndParseYamlConfig( + result.GetYamlConfig(), + result.GetVolatileYamlConfigs(), + RunConfig.Labels, + yamlConfig); + } + + RunConfig.InitialCmsConfig.CopyFrom(result.GetConfig()); + + RunConfig.InitialCmsYamlConfig.CopyFrom(yamlConfig); + NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), RunConfig.InitialCmsYamlConfig); + + if (yamlConfig.HasYamlConfigEnabled() && yamlConfig.GetYamlConfigEnabled()) { + appConfig = yamlConfig; + NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), appConfig); + } else { + appConfig = result.GetConfig(); + } if (RunConfig.PathToConfigCacheFile) { Cout << "Saving config to cache file " << RunConfig.PathToConfigCacheFile << Endl; diff --git a/ydb/core/driver_lib/run/CMakeLists.darwin.txt b/ydb/core/driver_lib/run/CMakeLists.darwin.txt index 9b9374ed6e..2cd5ee46dd 100644 --- a/ydb/core/driver_lib/run/CMakeLists.darwin.txt +++ b/ydb/core/driver_lib/run/CMakeLists.darwin.txt @@ -120,7 +120,7 @@ target_link_libraries(run PUBLIC lib-deprecated-client ydb-services-auth ydb-services-cms - ydb-services-console + ydb-services-dynamic_config ydb-services-datastreams ydb-services-discovery ydb-services-fq diff --git a/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt b/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt index 16064f2a3b..bbaee3b6af 100644 --- a/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt +++ b/ydb/core/driver_lib/run/CMakeLists.linux-aarch64.txt @@ -121,7 +121,7 @@ target_link_libraries(run PUBLIC lib-deprecated-client ydb-services-auth ydb-services-cms - ydb-services-console + ydb-services-dynamic_config ydb-services-datastreams ydb-services-discovery ydb-services-fq diff --git a/ydb/core/driver_lib/run/CMakeLists.linux.txt b/ydb/core/driver_lib/run/CMakeLists.linux.txt index 16064f2a3b..bbaee3b6af 100644 --- a/ydb/core/driver_lib/run/CMakeLists.linux.txt +++ b/ydb/core/driver_lib/run/CMakeLists.linux.txt @@ -121,7 +121,7 @@ target_link_libraries(run PUBLIC lib-deprecated-client ydb-services-auth ydb-services-cms - ydb-services-console + ydb-services-dynamic_config ydb-services-datastreams ydb-services-discovery ydb-services-fq diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index c22be5dce1..04ee376393 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -116,6 +116,9 @@ struct TKikimrRunConfig { TString ClusterName; + NKikimrConfig::TAppConfig InitialCmsConfig; + NKikimrConfig::TAppConfig InitialCmsYamlConfig; + TKikimrRunConfig(NKikimrConfig::TAppConfig& appConfig, ui32 nodeId = 0, const TKikimrScopeId& scopeId = {}); }; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index afda50b181..29278be5ad 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2498,11 +2498,13 @@ void THttpProxyServiceInitializer::InitializeServices(NActors::TActorSystemSetup TConfigsDispatcherInitializer::TConfigsDispatcherInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) , Labels(runConfig.Labels) + , InitialCmsConfig(runConfig.InitialCmsConfig) + , InitialCmsYamlConfig(runConfig.InitialCmsYamlConfig) { } void TConfigsDispatcherInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels); + IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels, InitialCmsConfig, InitialCmsYamlConfig); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( NConsole::MakeConfigsDispatcherID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index a9c6ba93ce..3717a6d0cf 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -447,6 +447,8 @@ public: private: TMap<TString, TString> Labels; + NKikimrConfig::TAppConfig InitialCmsConfig; + NKikimrConfig::TAppConfig InitialCmsYamlConfig; }; class TConfigsCacheInitializer : public IKikimrServicesInitializer { diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 48fbc246bb..f425617cc7 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -83,7 +83,7 @@ #include <ydb/services/auth/grpc_service.h> #include <ydb/services/cms/grpc_service.h> -#include <ydb/services/console/grpc_service.h> +#include <ydb/services/dynamic_config/grpc_service.h> #include <ydb/services/datastreams/grpc_service.h> #include <ydb/services/discovery/grpc_service.h> #include <ydb/services/fq/grpc_service.h> @@ -811,7 +811,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { server.AddService(new NGRpcService::TGRpcCmsService(ActorSystem.Get(), Counters, grpcRequestProxies[0], hasCms.IsRlAllowed())); - server.AddService(new NGRpcService::TGRpcConsoleService(ActorSystem.Get(), Counters, + server.AddService(new NGRpcService::TGRpcDynamicConfigService(ActorSystem.Get(), Counters, grpcRequestProxies[0], hasCms.IsRlAllowed())); } @@ -1104,6 +1104,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) AppData->SharedCacheConfig = runConfig.AppConfig.GetSharedCacheConfig(); } + if (runConfig.AppConfig.HasAwsCompatibilityConfig()) { + AppData->AwsCompatibilityConfig = runConfig.AppConfig.GetAwsCompatibilityConfig(); + } + // setup resource profiles AppData->ResourceProfiles = new TResourceProfiles; if (runConfig.AppConfig.GetBootstrapConfig().ResourceProfilesSize()) diff --git a/ydb/core/driver_lib/run/version.cpp b/ydb/core/driver_lib/run/version.cpp index 7cd90a34a8..d649b0d038 100644 --- a/ydb/core/driver_lib/run/version.cpp +++ b/ydb/core/driver_lib/run/version.cpp @@ -3,13 +3,13 @@ TMaybe<NActors::TInterconnectProxyCommon::TVersionInfo> VERSION = NActors::TInterconnectProxyCommon::TVersionInfo{ // version of this binary - "stable-23-1", + "stable-23-2", // compatible versions; must include all compatible old ones, including this one; version verification occurs on both // peers and connection is accepted if at least one of peers accepts the version of the other peer { "stable-23-1", - "stable-22-5" + "stable-23-2" } }; diff --git a/ydb/core/grpc_services/CMakeLists.darwin.txt b/ydb/core/grpc_services/CMakeLists.darwin.txt index e90b95ff36..209341cbd3 100644 --- a/ydb/core/grpc_services/CMakeLists.darwin.txt +++ b/ydb/core/grpc_services/CMakeLists.darwin.txt @@ -72,7 +72,7 @@ target_sources(ydb-core-grpc_services PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_cancel_operation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_cms.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_commit_transaction.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_console.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_dynamic_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_copy_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_copy_tables.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_export.cpp diff --git a/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt b/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt index e3eecd8806..fcf80c2b10 100644 --- a/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt +++ b/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt @@ -73,7 +73,7 @@ target_sources(ydb-core-grpc_services PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_cancel_operation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_cms.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_commit_transaction.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_console.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_dynamic_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_copy_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_copy_tables.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_export.cpp diff --git a/ydb/core/grpc_services/CMakeLists.linux.txt b/ydb/core/grpc_services/CMakeLists.linux.txt index e3eecd8806..fcf80c2b10 100644 --- a/ydb/core/grpc_services/CMakeLists.linux.txt +++ b/ydb/core/grpc_services/CMakeLists.linux.txt @@ -73,7 +73,7 @@ target_sources(ydb-core-grpc_services PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_cancel_operation.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_cms.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_commit_transaction.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_console.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_dynamic_config.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_copy_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_copy_tables.cpp ${CMAKE_SOURCE_DIR}/ydb/core/grpc_services/rpc_export.cpp diff --git a/ydb/core/grpc_services/grpc_request_check_actor.h b/ydb/core/grpc_services/grpc_request_check_actor.h index ad9458b285..45e24407c1 100644 --- a/ydb/core/grpc_services/grpc_request_check_actor.h +++ b/ydb/core/grpc_services/grpc_request_check_actor.h @@ -456,7 +456,9 @@ const TVector<TString>& TGrpcRequestCheckActor<TEvent>::GetPermissions() { static const TVector<TString> permissions = { "ydb.databases.list", "ydb.databases.create", - "ydb.databases.connect" + "ydb.databases.connect", + "ydb.tables.select", + "ydb.schemas.getMetadata" }; return permissions; } diff --git a/ydb/core/grpc_services/local_rpc/local_rpc.h b/ydb/core/grpc_services/local_rpc/local_rpc.h index 8bee988b85..86c23752f9 100644 --- a/ydb/core/grpc_services/local_rpc/local_rpc.h +++ b/ydb/core/grpc_services/local_rpc/local_rpc.h @@ -30,10 +30,14 @@ class TLocalRpcCtx : public NGRpcService::IRequestOpCtx { public: using TResp = typename TRpc::TResponse; template<typename TProto, typename TCb> - TLocalRpcCtx(TProto&& req, TCb&& cb, const TString& databaseName, const TString& token) + TLocalRpcCtx(TProto&& req, TCb&& cb, + const TString& databaseName, + const TString& token, + const TMaybe<TString>& requestType) : Request(std::forward<TProto>(req)) , CbWrapper(std::forward<TCb>(cb)) , DatabaseName(databaseName) + , RequestType(requestType) { InternalToken = new NACLib::TUserToken(token); } @@ -164,7 +168,7 @@ public: // Unimplemented methods void ReplyWithRpcStatus(grpc::StatusCode, const TString&, const TString&) override { - Y_FAIL("Unimplemented for local rpc"); + ReplyWithYdbStatus(Ydb::StatusIds::GENERIC_ERROR); } void SetStreamingNotify(NGrpc::IRequestContextBase::TOnNextReply&&) override { @@ -188,7 +192,7 @@ public: } const TMaybe<TString> GetRequestType() const override { - return Nothing(); + return RequestType; } void SetCostInfo(float consumed_units) override { @@ -222,6 +226,7 @@ private: typename TRpc::TRequest Request; TCbWrapper CbWrapper; const TString DatabaseName; + const TMaybe<TString> RequestType; TIntrusiveConstPtr<NACLib::TUserToken> InternalToken; const TString EmptySerializedTokenMessage_; @@ -232,13 +237,15 @@ private: }; template<typename TRpc> -NThreading::TFuture<typename TRpc::TResponse> DoLocalRpc(typename TRpc::TRequest&& proto, const TString& database, const TString& token, TActorSystem* actorSystem) { +NThreading::TFuture<typename TRpc::TResponse> DoLocalRpc(typename TRpc::TRequest&& proto, const TString& database, + const TString& token, const TMaybe<TString>& requestType, TActorSystem* actorSystem) +{ auto promise = NThreading::NewPromise<typename TRpc::TResponse>(); proto.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); using TCbWrapper = TPromiseWrapper<typename TRpc::TResponse>; - auto req = new TLocalRpcCtx<TRpc, TCbWrapper>(std::move(proto), TCbWrapper(promise), database, token); + auto req = new TLocalRpcCtx<TRpc, TCbWrapper>(std::move(proto), TCbWrapper(promise), database, token, requestType); auto actor = TRpc::CreateRpcActor(req); actorSystem->Register(actor, TMailboxType::HTSwap, actorSystem->AppData<TAppData>()->UserPoolId); @@ -246,13 +253,25 @@ NThreading::TFuture<typename TRpc::TResponse> DoLocalRpc(typename TRpc::TRequest } template<typename TRpc> -TActorId DoLocalRpcSameMailbox(typename TRpc::TRequest&& proto, std::function<void(typename TRpc::TResponse)>&& cb, const TString& database, const TString& token, const TActorContext& ctx) { +NThreading::TFuture<typename TRpc::TResponse> DoLocalRpc(typename TRpc::TRequest&& proto, const TString& database, const TString& token, TActorSystem* actorSystem) { + return DoLocalRpc<TRpc>(std::move(proto), database, token, Nothing(), actorSystem); +} + +template<typename TRpc> +TActorId DoLocalRpcSameMailbox(typename TRpc::TRequest&& proto, std::function<void(typename TRpc::TResponse)>&& cb, + const TString& database, const TString& token, const TMaybe<TString>& requestType, const TActorContext& ctx) +{ proto.mutable_operation_params()->set_operation_mode(Ydb::Operations::OperationParams::SYNC); - auto req = new TLocalRpcCtx<TRpc, std::function<void(typename TRpc::TResponse)>>(std::move(proto), std::move(cb), database, token); + auto req = new TLocalRpcCtx<TRpc, std::function<void(typename TRpc::TResponse)>>(std::move(proto), std::move(cb), database, token, requestType); auto actor = TRpc::CreateRpcActor(req); return ctx.RegisterWithSameMailbox(actor); } +template<typename TRpc> +TActorId DoLocalRpcSameMailbox(typename TRpc::TRequest&& proto, std::function<void(typename TRpc::TResponse)>&& cb, const TString& database, const TString& token, const TActorContext& ctx) { + return DoLocalRpcSameMailbox<TRpc>(std::move(proto), std::move(cb), database, token, Nothing(), ctx); +} + } // namespace NRpcService } // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_console.cpp b/ydb/core/grpc_services/rpc_console.cpp deleted file mode 100644 index c8ee3e5820..0000000000 --- a/ydb/core/grpc_services/rpc_console.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include "service_console.h" -#include "rpc_deferrable.h" - -#include <ydb/core/grpc_services/base/base.h> -#include <ydb/core/base/appdata.h> -#include <ydb/core/base/tablet_pipe.h> -#include <ydb/core/cms/console/console.h> -#include <ydb/public/api/protos/draft/ydb_console.pb.h> - -namespace NKikimr::NGRpcService { - -using namespace NActors; -using namespace NConsole; -using namespace Ydb; - -using TEvApplyConfigRequest = TGrpcRequestOperationCall<Console::ApplyConfigRequest, - Console::ApplyConfigResponse>; - -using TEvAddVolatileConfigRequest = TGrpcRequestOperationCall<Console::AddVolatileConfigRequest, - Console::AddVolatileConfigResponse>; - -using TEvRemoveVolatileConfigRequest = TGrpcRequestOperationCall<Console::RemoveVolatileConfigRequest, - Console::RemoveVolatileConfigResponse>; - -using TEvGetConfigRequest = TGrpcRequestOperationCall<Console::GetConfigRequest, - Console::GetConfigResponse>; - -using TEvResolveConfigRequest = TGrpcRequestOperationCall<Console::ResolveConfigRequest, - Console::ResolveConfigResponse>; - -using TEvResolveAllConfigRequest = TGrpcRequestOperationCall<Console::ResolveAllConfigRequest, - Console::ResolveAllConfigResponse>; - -template <typename TRequest, typename TConsoleRequest, typename TConsoleResponse> -class TConsoleRPC : public TRpcOperationRequestActor<TConsoleRPC<TRequest, TConsoleRequest, TConsoleResponse>, TRequest> { - using TThis = TConsoleRPC<TRequest, TConsoleRequest, TConsoleResponse>; - using TBase = TRpcOperationRequestActor<TThis, TRequest>; - - TActorId ConsolePipe; - -public: - TConsoleRPC(IRequestOpCtx* msg) - : TBase(msg) - { - } - - void Bootstrap() - { - TBase::Bootstrap(TActivationContext::AsActorContext()); - - auto dinfo = AppData()->DomainsInfo; - auto domain = dinfo->Domains.begin()->second; - ui32 group = dinfo->GetDefaultStateStorageGroup(domain->DomainUid); - - NTabletPipe::TClientConfig pipeConfig; - pipeConfig.RetryPolicy = { - .RetryLimitCount = 10, - }; - auto pipe = NTabletPipe::CreateClient(IActor::SelfId(), MakeConsoleID(group), pipeConfig); - ConsolePipe = IActor::RegisterWithSameMailbox(pipe); - - SendRequest(); - - this->Become(&TThis::StateWork); - } - -private: - void PassAway() - { - NTabletPipe::CloseClient(IActor::SelfId(), ConsolePipe); - TBase::PassAway(); - } - - template<typename T> - void HandleWithOperationParams(T& ev) - { - const auto& response = ev->Get()->Record.GetResponse(); - if (response.operation().ready() == false - && this->GetProtoRequest()->operation_params().operation_mode() == Ydb::Operations::OperationParams::SYNC) { - auto request = MakeHolder<TEvConsole::TEvNotifyOperationCompletionRequest>(); - request->Record.MutableRequest()->set_id(response.operation().id()); - request->Record.SetUserToken(this->Request_->GetSerializedToken()); - - NTabletPipe::SendData(IActor::SelfId(), ConsolePipe, request.Release()); - } else { - return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, response, TActivationContext::AsActorContext()); - } - } - - void Handle(TEvConsole::TEvGetAllConfigsResponse::TPtr& ev) - { - HandleWithOperationParams(ev); - } - - void Handle(TEvConsole::TEvResolveConfigResponse::TPtr& ev) - { - HandleWithOperationParams(ev); - } - - void Handle(TEvConsole::TEvResolveAllConfigResponse::TPtr& ev) - { - HandleWithOperationParams(ev); - } - - template<typename T> - void Handle(T& ev) - { - auto& response = ev->Get()->Record.GetResponse(); - TProtoResponseHelper::SendProtoResponse(response, response.operation().status(), this->Request_); - PassAway(); - } - - void Handle(TEvConsole::TEvOperationCompletionNotification::TPtr& ev) - { - this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); - PassAway(); - } - - void Handle(TEvConsole::TEvNotifyOperationCompletionResponse::TPtr& ev) - { - if (ev->Get()->Record.GetResponse().operation().ready() == true) { - this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); - PassAway(); - } - } - - void Undelivered() - { - this->Request_->RaiseIssue(NYql::TIssue("Console is unavailable")); - this->Request_->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); - PassAway(); - } - - void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev) noexcept - { - if (ev->Get()->Status != NKikimrProto::OK) - Undelivered(); - } - - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) - { - switch (ev->GetTypeRewrite()) { - hFunc(TConsoleResponse, Handle); - cFunc(TEvTabletPipe::EvClientDestroyed, Undelivered); - hFunc(TEvTabletPipe::TEvClientConnected, Handle); - hFunc(TEvConsole::TEvOperationCompletionNotification, Handle); - hFunc(TEvConsole::TEvNotifyOperationCompletionResponse, Handle); - default: TBase::StateFuncBase(ev, ctx); - } - } - - void SendRequest() - { - auto request = MakeHolder<TConsoleRequest>(); - request->Record.MutableRequest()->CopyFrom(*this->GetProtoRequest()); - request->Record.SetUserToken(this->Request_->GetSerializedToken()); - NTabletPipe::SendData(IActor::SelfId(), ConsolePipe, request.Release()); - } -}; - -void DoApplyConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { - TActivationContext::AsActorContext().Register( - new TConsoleRPC<TEvApplyConfigRequest, - TEvConsole::TEvApplyConfigRequest, - TEvConsole::TEvApplyConfigResponse>(p.release())); -} - -void DoAddVolatileConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { - TActivationContext::AsActorContext().Register( - new TConsoleRPC<TEvAddVolatileConfigRequest, - TEvConsole::TEvAddVolatileConfigRequest, - TEvConsole::TEvAddVolatileConfigResponse>(p.release())); -} - -void DoRemoveVolatileConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { - TActivationContext::AsActorContext().Register( - new TConsoleRPC<TEvRemoveVolatileConfigRequest, - TEvConsole::TEvRemoveVolatileConfigRequest, - TEvConsole::TEvRemoveVolatileConfigResponse>(p.release())); -} - -void DoGetConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { - TActivationContext::AsActorContext().Register( - new TConsoleRPC<TEvGetConfigRequest, - TEvConsole::TEvGetAllConfigsRequest, - TEvConsole::TEvGetAllConfigsResponse>(p.release())); -} - -void DoResolveConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { - TActivationContext::AsActorContext().Register( - new TConsoleRPC<TEvResolveConfigRequest, - TEvConsole::TEvResolveConfigRequest, - TEvConsole::TEvResolveConfigResponse>(p.release())); -} - -void DoResolveAllConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { - TActivationContext::AsActorContext().Register( - new TConsoleRPC<TEvResolveAllConfigRequest, - TEvConsole::TEvResolveAllConfigRequest, - TEvConsole::TEvResolveAllConfigResponse>(p.release())); -} - -} // namespace NKikimr::NGRpcService diff --git a/ydb/core/grpc_services/rpc_dynamic_config.cpp b/ydb/core/grpc_services/rpc_dynamic_config.cpp new file mode 100644 index 0000000000..5db8ce428d --- /dev/null +++ b/ydb/core/grpc_services/rpc_dynamic_config.cpp @@ -0,0 +1,278 @@ +#include "service_dynamic_config.h" +#include "rpc_deferrable.h" + +#include <ydb/core/grpc_services/base/base.h> +#include <ydb/core/base/appdata.h> +#include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/cms/console/console.h> +#include <ydb/public/api/protos/draft/ydb_dynamic_config.pb.h> + +namespace NKikimr::NGRpcService { + +using namespace NActors; +using namespace NConsole; +using namespace Ydb; + +using TEvSetConfigRequest = TGrpcRequestOperationCall<DynamicConfig::SetConfigRequest, + DynamicConfig::SetConfigResponse>; + +using TEvReplaceConfigRequest = TGrpcRequestOperationCall<DynamicConfig::ReplaceConfigRequest, + DynamicConfig::ReplaceConfigResponse>; + +using TEvDropConfigRequest = TGrpcRequestOperationCall<DynamicConfig::DropConfigRequest, + DynamicConfig::DropConfigResponse>; + +using TEvAddVolatileConfigRequest = TGrpcRequestOperationCall<DynamicConfig::AddVolatileConfigRequest, + DynamicConfig::AddVolatileConfigResponse>; + +using TEvRemoveVolatileConfigRequest = TGrpcRequestOperationCall<DynamicConfig::RemoveVolatileConfigRequest, + DynamicConfig::RemoveVolatileConfigResponse>; + +using TEvGetNodeLabelsRequest = TGrpcRequestOperationCall<DynamicConfig::GetNodeLabelsRequest, + DynamicConfig::GetNodeLabelsResponse>; + +using TEvGetMetadataRequest = TGrpcRequestOperationCall<DynamicConfig::GetMetadataRequest, + DynamicConfig::GetMetadataResponse>; + +using TEvGetConfigRequest = TGrpcRequestOperationCall<DynamicConfig::GetConfigRequest, + DynamicConfig::GetConfigResponse>; + +using TEvResolveConfigRequest = TGrpcRequestOperationCall<DynamicConfig::ResolveConfigRequest, + DynamicConfig::ResolveConfigResponse>; + +using TEvResolveAllConfigRequest = TGrpcRequestOperationCall<DynamicConfig::ResolveAllConfigRequest, + DynamicConfig::ResolveAllConfigResponse>; + +template <typename TRequest, typename TConsoleRequest, typename TConsoleResponse> +class TDynamicConfigRPC : public TRpcOperationRequestActor<TDynamicConfigRPC<TRequest, TConsoleRequest, TConsoleResponse>, TRequest> { + using TThis = TDynamicConfigRPC<TRequest, TConsoleRequest, TConsoleResponse>; + using TBase = TRpcOperationRequestActor<TThis, TRequest>; + + TActorId ConsolePipe; + +public: + TDynamicConfigRPC(IRequestOpCtx* msg) + : TBase(msg) + { + } + + void Bootstrap() + { + TBase::Bootstrap(TActivationContext::AsActorContext()); + + auto dinfo = AppData()->DomainsInfo; + auto domain = dinfo->Domains.begin()->second; + ui32 group = dinfo->GetDefaultStateStorageGroup(domain->DomainUid); + + NTabletPipe::TClientConfig pipeConfig; + pipeConfig.RetryPolicy = { + .RetryLimitCount = 10, + }; + auto pipe = NTabletPipe::CreateClient(IActor::SelfId(), MakeConsoleID(group), pipeConfig); + ConsolePipe = IActor::RegisterWithSameMailbox(pipe); + + SendRequest(); + + this->Become(&TThis::StateWork); + } + +private: + void PassAway() + { + NTabletPipe::CloseClient(IActor::SelfId(), ConsolePipe); + TBase::PassAway(); + } + + template<typename T> + void HandleWithOperationParams(T& ev) + { + const auto& response = ev->Get()->Record.GetResponse(); + if (response.operation().ready() == false + && this->GetProtoRequest()->operation_params().operation_mode() == Ydb::Operations::OperationParams::SYNC) { + auto request = MakeHolder<TEvConsole::TEvNotifyOperationCompletionRequest>(); + request->Record.MutableRequest()->set_id(response.operation().id()); + request->Record.SetUserToken(this->Request_->GetSerializedToken()); + + NTabletPipe::SendData(IActor::SelfId(), ConsolePipe, request.Release()); + } else if (response.operation().status() != Ydb::StatusIds::SUCCESS) { + return TBase::Reply(response.operation().status(), response.operation().issues(), TActivationContext::AsActorContext()); + } else { + return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, response, TActivationContext::AsActorContext()); + } + } + + void Handle(TEvConsole::TEvGetAllMetadataResponse::TPtr& ev) + { + return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvGetAllConfigsResponse::TPtr& ev) + { + return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvResolveConfigResponse::TPtr& ev) + { + return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvResolveAllConfigResponse::TPtr& ev) + { + return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvGetNodeLabelsResponse::TPtr& ev) { + return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvUnauthorized::TPtr&) { + ::google::protobuf::RepeatedPtrField< ::Ydb::Issue::IssueMessage> issues; + auto issue = issues.Add(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message("User must have administrator rights"); + + return TBase::Reply(Ydb::StatusIds::UNAUTHORIZED, issues, TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvDisabled::TPtr&) { + ::google::protobuf::RepeatedPtrField< ::Ydb::Issue::IssueMessage> issues; + auto issue = issues.Add(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message("Feature is disabled"); + + return TBase::Reply(Ydb::StatusIds::BAD_REQUEST, issues, TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvGenericError::TPtr& ev) { + return TBase::Reply(ev->Get()->Record.GetYdbStatus(), ev->Get()->Record.GetIssues(), TActivationContext::AsActorContext()); + } + + template<typename T> + void Handle(T& ev) + { + Y_UNUSED(ev); + TBase::Reply(Ydb::StatusIds::SUCCESS, TActivationContext::AsActorContext()); + } + + void Handle(TEvConsole::TEvOperationCompletionNotification::TPtr& ev) + { + this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); + PassAway(); + } + + void Handle(TEvConsole::TEvNotifyOperationCompletionResponse::TPtr& ev) + { + if (ev->Get()->Record.GetResponse().operation().ready() == true) { + this->Request_->SendOperation(ev->Get()->Record.GetResponse().operation()); + PassAway(); + } + } + + void Undelivered() + { + this->Request_->RaiseIssue(NYql::TIssue("Console is unavailable")); + this->Request_->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE); + PassAway(); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev) noexcept + { + if (ev->Get()->Status != NKikimrProto::OK) + Undelivered(); + } + + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) + { + switch (ev->GetTypeRewrite()) { + hFunc(TConsoleResponse, Handle); + cFunc(TEvTabletPipe::EvClientDestroyed, Undelivered); + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvConsole::TEvOperationCompletionNotification, Handle); + hFunc(TEvConsole::TEvNotifyOperationCompletionResponse, Handle); + hFunc(TEvConsole::TEvUnauthorized, Handle); + hFunc(TEvConsole::TEvDisabled, Handle); + hFunc(TEvConsole::TEvGenericError, Handle); + default: TBase::StateFuncBase(ev, ctx); + } + } + + void SendRequest() + { + auto request = MakeHolder<TConsoleRequest>(); + request->Record.MutableRequest()->CopyFrom(*this->GetProtoRequest()); + request->Record.SetUserToken(this->Request_->GetSerializedToken()); + NTabletPipe::SendData(IActor::SelfId(), ConsolePipe, request.Release()); + } +}; + +void DoSetConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvSetConfigRequest, + TEvConsole::TEvSetYamlConfigRequest, + TEvConsole::TEvSetYamlConfigResponse>(p.release())); +} + +void DoReplaceConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvReplaceConfigRequest, + TEvConsole::TEvReplaceYamlConfigRequest, + TEvConsole::TEvReplaceYamlConfigResponse>(p.release())); +} + +void DoDropConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvDropConfigRequest, + TEvConsole::TEvDropConfigRequest, + TEvConsole::TEvDropConfigResponse>(p.release())); +} + +void DoAddVolatileConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvAddVolatileConfigRequest, + TEvConsole::TEvAddVolatileConfigRequest, + TEvConsole::TEvAddVolatileConfigResponse>(p.release())); +} + +void DoRemoveVolatileConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvRemoveVolatileConfigRequest, + TEvConsole::TEvRemoveVolatileConfigRequest, + TEvConsole::TEvRemoveVolatileConfigResponse>(p.release())); +} + +void DoGetNodeLabelsRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvGetNodeLabelsRequest, + TEvConsole::TEvGetNodeLabelsRequest, + TEvConsole::TEvGetNodeLabelsResponse>(p.release())); +} + +void DoGetMetadataRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvGetMetadataRequest, + TEvConsole::TEvGetAllMetadataRequest, + TEvConsole::TEvGetAllMetadataResponse>(p.release())); +} + +void DoGetConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvGetConfigRequest, + TEvConsole::TEvGetAllConfigsRequest, + TEvConsole::TEvGetAllConfigsResponse>(p.release())); +} + +void DoResolveConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvResolveConfigRequest, + TEvConsole::TEvResolveConfigRequest, + TEvConsole::TEvResolveConfigResponse>(p.release())); +} + +void DoResolveAllConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) { + TActivationContext::AsActorContext().Register( + new TDynamicConfigRPC<TEvResolveAllConfigRequest, + TEvConsole::TEvResolveAllConfigRequest, + TEvConsole::TEvResolveAllConfigResponse>(p.release())); +} + +} // namespace NKikimr::NGRpcService diff --git a/ydb/core/grpc_services/service_console.h b/ydb/core/grpc_services/service_dynamic_config.h index 0db78724a5..891768b73a 100644 --- a/ydb/core/grpc_services/service_console.h +++ b/ydb/core/grpc_services/service_dynamic_config.h @@ -8,14 +8,22 @@ namespace NGRpcService { class IRequestOpCtx; class IFacilityProvider; -void DoApplyConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); +void DoGetNodeLabelsRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +void DoGetMetadataRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +void DoGetConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +void DoSetConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +void DoReplaceConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); + +void DoDropConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); void DoAddVolatileConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); void DoRemoveVolatileConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); -void DoGetConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); - void DoResolveConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); void DoResolveAllConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&); diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index baba609122..6497473e8f 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -346,6 +346,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.EnableKqpScanQueryStreamIdxLookupJoin = serviceConfig.GetEnableKqpScanQueryStreamIdxLookupJoin(); kqpConfig.EnablePredicateExtractForDataQuery = serviceConfig.GetEnablePredicateExtractForDataQueries(); kqpConfig.EnablePredicateExtractForScanQuery = serviceConfig.GetEnablePredicateExtractForScanQueries(); + kqpConfig.EnableKqpImmediateEffects = serviceConfig.GetEnableKqpImmediateEffects(); } IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings, diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index dc9b0eedd3..5388386fa8 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -366,13 +366,16 @@ private: bool enableKqpDataQueryPredicateExtract = Config.GetEnablePredicateExtractForDataQueries(); bool enableKqpScanQueryPredicateExtract = Config.GetEnablePredicateExtractForScanQueries(); + bool defaultSyntaxVersion = Config.GetSqlVersion(); + Config.Swap(event.MutableConfig()->MutableTableServiceConfig()); LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Updated config"); auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event); Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); - if (Config.GetEnableKqpDataQueryStreamLookup() != enableKqpDataQueryStreamLookup || + if (Config.GetSqlVersion() != defaultSyntaxVersion || + Config.GetEnableKqpDataQueryStreamLookup() != enableKqpDataQueryStreamLookup || Config.GetEnableKqpScanQueryStreamLookup() != enableKqpScanQueryStreamLookup || Config.GetEnableKqpScanQueryStreamIdxLookupJoin() != enableKqpScanQueryStreamIdxLookupJoin || Config.GetEnableKqpDataQuerySourceRead() != enableKqpDataQuerySourceRead || diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index 455ff0c9c8..561dccef84 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -138,6 +138,8 @@ public: if (Request.Snapshot.IsValid()) { YQL_ENSURE(Request.IsolationLevel == NKikimrKqp::ISOLATION_LEVEL_SERIALIZABLE); } + + ReadOnlyTx = IsReadOnlyTx(); } void CheckExecutionComplete() { @@ -181,6 +183,31 @@ public: } } + bool ForceAcquireSnapshot() const { + const bool forceSnapshot = ( + ReadOnlyTx && + !ImmediateTx && + !HasPersistentChannels && + (!Database.empty() || AppData()->EnableMvccSnapshotWithLegacyDomainRoot) && + AppData()->FeatureFlags.GetEnableMvccSnapshotReads() + ); + + return forceSnapshot; + } + + bool GetUseFollowers() const { + return ( + // first, we must specify read stale flag. + Request.IsolationLevel == NKikimrKqp::ISOLATION_LEVEL_READ_STALE && + // next, if snapshot is already defined, so in this case followers are not allowed. + !Snapshot.IsValid() && + // ensure that followers are allowed only for read only transactions. + ReadOnlyTx && + // if we are forced to acquire snapshot by some reason, so we cannot use followers. + !ForceAcquireSnapshot() + ); + } + void Finalize() { if (LocksBroken) { TString message = "Transaction locks invalidated."; @@ -979,7 +1006,7 @@ private: LOG_I("Reattach to shard " << tabletId); - Send(MakePipePeNodeCacheID(UseFollowers), new TEvPipeCache::TEvForward( + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward( new TEvDataShard::TEvProposeTransactionAttach(tabletId, TxId), tabletId, /* subscribe */ true), 0, ++shardState->ReattachState.Cookie); } @@ -1113,6 +1140,28 @@ private: } private: + bool IsReadOnlyTx() const { + if (Request.TopicOperations.HasOperations()) { + YQL_ENSURE(!Request.UseImmediateEffects); + return false; + } + + if (Request.ValidateLocks && Request.EraseLocks) { + YQL_ENSURE(!Request.UseImmediateEffects); + return false; + } + + for (const auto& tx : Request.Transactions) { + for (const auto& stage : tx.Body->GetStages()) { + if (stage.GetIsEffectsStage()) { + return false; + } + } + } + + return true; + } + void FillGeneralReadInfo(TTaskMeta& taskMeta, ui64 itemsLimit, bool reverse) { if (taskMeta.Reads && !taskMeta.Reads.GetRef().empty()) { // Validate parameters @@ -1332,7 +1381,7 @@ private: TShardState shardState; shardState.State = ImmediateTx ? TShardState::EState::Executing : TShardState::EState::Preparing; shardState.DatashardState.ConstructInPlace(); - shardState.DatashardState->Follower = UseFollowers; + shardState.DatashardState->Follower = GetUseFollowers(); if (Deadline) { TDuration timeout = *Deadline - TAppData::TimeProvider->Now(); @@ -1378,7 +1427,7 @@ private: (ImmediateTx ? NTxDataShard::TTxFlags::Immediate : 0) | (VolatileTx ? NTxDataShard::TTxFlags::VolatilePrepare : 0); TEvDataShard::TEvProposeTransaction* ev; - if (Snapshot.IsValid() && (ReadOnlyTx || AppData()->FeatureFlags.GetEnableKqpImmediateEffects())) { + if (Snapshot.IsValid() && (ReadOnlyTx || Request.UseImmediateEffects)) { ev = new TEvDataShard::TEvProposeTransaction( NKikimrTxDataShard::TX_KIND_DATA, SelfId(), @@ -1400,7 +1449,7 @@ private: LOG_D("ExecuteDatashardTransaction traceId.verbosity: " << std::to_string(traceId.GetVerbosity())); - Send(MakePipePeNodeCacheID(UseFollowers), new TEvPipeCache::TEvForward(ev, shardId, true), 0, 0, std::move(traceId)); + Send(MakePipePeNodeCacheID(GetUseFollowers()), new TEvPipeCache::TEvForward(ev, shardId, true), 0, 0, std::move(traceId)); auto result = ShardStates.emplace(shardId, std::move(shardState)); YQL_ENSURE(result.second); @@ -1453,7 +1502,6 @@ private: LWTRACK(KqpDataExecuterStartExecute, ResponseEv->Orbit, TxId); size_t readActors = 0; - ReadOnlyTx = !Request.TopicOperations.HasOperations(); for (ui32 txIdx = 0; txIdx < Request.Transactions.size(); ++txIdx) { auto& tx = Request.Transactions[txIdx]; @@ -1482,8 +1530,6 @@ private: LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst()); - ReadOnlyTx &= !stage.GetIsEffectsStage(); - if (stage.SourcesSize() > 0) { switch (stage.GetSources(0).GetTypeCase()) { case NKqpProto::TKqpSource::kReadRangesSource: @@ -1680,17 +1726,9 @@ private: // Single-shard transactions are always immediate ImmediateTx = (datashardTxs.size() + Request.TopicOperations.GetSize() + readActors) <= 1 && !HasStreamLookup; - if (ImmediateTx) { - // Transaction cannot be both immediate and volatile - YQL_ENSURE(!VolatileTx); - } - switch (Request.IsolationLevel) { // OnlineRO with AllowInconsistentReads = true case NKikimrKqp::ISOLATION_LEVEL_READ_UNCOMMITTED: - // StaleRO transactions always execute as immediate - // (legacy behaviour, for compatibility with current execution engine) - case NKikimrKqp::ISOLATION_LEVEL_READ_STALE: YQL_ENSURE(ReadOnlyTx); YQL_ENSURE(!VolatileTx); ImmediateTx = true; @@ -1700,8 +1738,14 @@ private: break; } - if ((ReadOnlyTx || AppData()->FeatureFlags.GetEnableKqpImmediateEffects()) && Request.Snapshot.IsValid()) { + if (ImmediateTx) { + // Transaction cannot be both immediate and volatile + YQL_ENSURE(!VolatileTx); + } + + if ((ReadOnlyTx || Request.UseImmediateEffects) && Request.Snapshot.IsValid()) { // Snapshot reads are always immediate + // Uncommitted writes are executed without coordinators, so they can be immediate YQL_ENSURE(!VolatileTx); Snapshot = Request.Snapshot; ImmediateTx = true; @@ -1722,7 +1766,8 @@ private: shardIds.insert(shardId); } - auto kqpShardsResolver = CreateKqpShardsResolver(SelfId(), TxId, std::move(shardIds)); + auto kqpShardsResolver = CreateKqpShardsResolver( + SelfId(), TxId, GetUseFollowers(), std::move(shardIds)); RegisterWithSameMailbox(kqpShardsResolver); Become(&TKqpDataExecuter::WaitResolveState); } else { @@ -1741,14 +1786,7 @@ private: } void OnShardsResolve() { - const bool forceSnapshot = ( - ReadOnlyTx && - !ImmediateTx && - !HasPersistentChannels && - (!Database.empty() || AppData()->EnableMvccSnapshotWithLegacyDomainRoot) && - AppData()->FeatureFlags.GetEnableMvccSnapshotReads()); - - if (forceSnapshot) { + if (ForceAcquireSnapshot()) { YQL_ENSURE(!VolatileTx); auto longTxService = NLongTxService::MakeLongTxServiceID(SelfId().NodeId()); Send(longTxService, new NLongTxService::TEvLongTxService::TEvAcquireReadSnapshot(Database)); @@ -1800,21 +1838,6 @@ private: using TTopicTabletTxs = THashMap<ui64, NKikimrPQ::TKqpTransaction>; void ContinueExecute() { - UseFollowers = Request.IsolationLevel == NKikimrKqp::ISOLATION_LEVEL_READ_STALE; - - if (!ImmediateTx) { - // Followers only allowed for single shard transactions. - // (legacy behaviour, for compatibility with current execution engine) - UseFollowers = false; - } - if (Snapshot.IsValid()) { - // TODO: KIKIMR-11912 - UseFollowers = false; - } - if (UseFollowers) { - YQL_ENSURE(ReadOnlyTx); - } - if (Stats) { //Stats->AffectedShards = datashardTxs.size(); Stats->DatashardStats.reserve(DatashardTxs.size()); @@ -1934,13 +1957,21 @@ private: if (!locksList.empty()) { auto* protoLocks = tx.MutableLocks()->MutableLocks(); protoLocks->Reserve(locksList.size()); + bool hasWrites = false; for (auto& lock : locksList) { + hasWrites = hasWrites || lock.GetHasWrites(); protoLocks->Add()->Swap(&lock); } if (needCommit) { // We also send the result on commit sendingShardsSet.insert(shardId); + + if (hasWrites) { + // Tx with uncommitted changes can be aborted due to conflicts, + // so shards with write locks should receive readsets + receivingShardsSet.insert(shardId); + } } } } @@ -2001,11 +2032,12 @@ private: NWilson::TSpan sendTasksSpan(TWilsonKqp::DataExecuterSendTasksAndTxs, ExecuterStateSpan.GetTraceId(), "SendTasksAndTxs", NWilson::EFlags::AUTO_END); LWTRACK(KqpDataExecuterStartTasksAndTxs, ResponseEv->Orbit, TxId, ComputeTasks.size(), DatashardTxs.size()); + const bool useFollowers = GetUseFollowers(); // first, start compute tasks TVector<ui64> computeTaskIds{Reserve(ComputeTasks.size())}; for (auto&& taskDesc : ComputeTasks) { computeTaskIds.emplace_back(taskDesc.GetId()); - FillInputSettings(taskDesc); + FillInputSettings(taskDesc, useFollowers); ExecuteDataComputeTask(std::move(taskDesc)); } @@ -2039,7 +2071,7 @@ private: } } remoteComputeTasksCnt += 1; - FillInputSettings(taskDesc); + FillInputSettings(taskDesc, useFollowers); PendingComputeTasks.insert(taskDesc.GetId()); tasksPerNode[it->second].emplace_back(std::move(taskDesc)); } @@ -2113,7 +2145,7 @@ private: << ", volatile: " << VolatileTx << ", immediate: " << ImmediateTx << ", remote tasks" << remoteComputeTasksCnt - << ", useFollowers: " << UseFollowers); + << ", useFollowers: " << GetUseFollowers()); LOG_T("Updating channels after the creation of compute actors"); THashMap<TActorId, THashSet<ui64>> updates; @@ -2154,7 +2186,7 @@ private: LOG_D("Executing KQP transaction on topic tablet: " << tabletId << ", lockTxId: " << lockTxId); - Send(MakePipePeNodeCacheID(UseFollowers), + Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.release(), tabletId, true), 0, 0, @@ -2164,7 +2196,7 @@ private: state.State = ImmediateTx ? TShardState::EState::Executing : TShardState::EState::Preparing; state.DatashardState.ConstructInPlace(); - state.DatashardState->Follower = UseFollowers; + state.DatashardState->Follower = false; state.DatashardState->ShardReadLocks = Request.TopicOperations.TabletHasReadOperations(tabletId); @@ -2183,7 +2215,7 @@ private: Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - if (UseFollowers) { + if (GetUseFollowers()) { Send(MakePipePeNodeCacheID(true), new TEvPipeCache::TEvUnlink(0)); } @@ -2241,7 +2273,7 @@ private: } } - void FillInputSettings(NYql::NDqProto::TDqTask& task) { + void FillInputSettings(NYql::NDqProto::TDqTask& task, bool useFollowers) { for (auto& input : *task.MutableInputs()) { if (input.HasTransform()) { auto transform = input.MutableTransform(); @@ -2268,7 +2300,7 @@ private: settings.SetImmediateTx(ImmediateTx); transform->MutableSettings()->PackFrom(settings); } - if (input.HasSource() && Snapshot != Request.Snapshot && input.GetSource().GetType() == NYql::KqpReadRangesSourceName) { + if (input.HasSource() && (Snapshot != Request.Snapshot || useFollowers) && input.GetSource().GetType() == NYql::KqpReadRangesSourceName) { auto source = input.MutableSource(); const google::protobuf::Any& settingsAny = source->GetSettings(); @@ -2284,6 +2316,10 @@ private: settings.MutableSnapshot()->SetTxId(Snapshot.TxId); } + if (useFollowers) { + settings.SetUseFollowers(useFollowers); + } + source->MutableSettings()->PackFrom(settings); } } @@ -2328,7 +2364,6 @@ private: bool ReadOnlyTx = true; bool VolatileTx = false; bool ImmediateTx = false; - bool UseFollowers = false; bool TxPlanned = false; bool LocksBroken = false; diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index 9ca58d1f38..7da9ef93cc 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -719,7 +719,10 @@ protected: auto& stage = GetStage(stageInfo); YQL_ENSURE(stage.GetSources(0).HasReadRangesSource()); - YQL_ENSURE(stage.InputsSize() == 0 && stage.SourcesSize() == 1, "multiple sources or sources mixed with connections"); + YQL_ENSURE(stage.GetSources(0).GetInputIndex() == 0 && stage.SourcesSize() == 1); + for (auto& input : stage.inputs()) { + YQL_ENSURE(input.HasBroadcast()); + } auto& source = stage.GetSources(0).GetReadRangesSource(); diff --git a/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp b/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp index 19fbb4cd60..6073aea73f 100644 --- a/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp +++ b/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp @@ -24,6 +24,7 @@ void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShar setMemberDataType(*structType.AddMember(), "LockId", NKikimr::NUdf::TDataType<ui64>::Id); setMemberDataType(*structType.AddMember(), "PathId", NKikimr::NUdf::TDataType<ui64>::Id); setMemberDataType(*structType.AddMember(), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id); + setMemberDataType(*structType.AddMember(), "HasWrites", NKikimr::NUdf::TDataType<bool>::Id); auto& value = *result.MutableValue(); for (auto& lock : locks) { @@ -34,6 +35,7 @@ void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShar item.AddStruct()->SetUint64(lock.GetLockId()); item.AddStruct()->SetUint64(lock.GetPathId()); item.AddStruct()->SetUint64(lock.GetSchemeShard()); + item.AddStruct()->SetBool(lock.GetHasWrites()); } } @@ -50,13 +52,14 @@ NKikimrTxDataShard::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock) { YQL_ENSURE(type.GetKind() == NKikimrMiniKQL::ETypeKind::Struct); auto& structType = type.GetStruct(); - YQL_ENSURE(structType.MemberSize() == 6); + YQL_ENSURE(structType.MemberSize() == 7); ensureMemberDataType(structType.GetMember(0), "Counter", NKikimr::NUdf::TDataType<ui64>::Id); ensureMemberDataType(structType.GetMember(1), "DataShard", NKikimr::NUdf::TDataType<ui64>::Id); ensureMemberDataType(structType.GetMember(2), "Generation", NKikimr::NUdf::TDataType<ui32>::Id); ensureMemberDataType(structType.GetMember(3), "LockId", NKikimr::NUdf::TDataType<ui64>::Id); ensureMemberDataType(structType.GetMember(4), "PathId", NKikimr::NUdf::TDataType<ui64>::Id); ensureMemberDataType(structType.GetMember(5), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id); + ensureMemberDataType(structType.GetMember(6), "HasWrites", NKikimr::NUdf::TDataType<bool>::Id); NKikimrTxDataShard::TLock dsLock; dsLock.SetCounter(value.GetStruct(0).GetUint64()); @@ -65,6 +68,7 @@ NKikimrTxDataShard::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock) { dsLock.SetLockId(value.GetStruct(3).GetUint64()); dsLock.SetPathId(value.GetStruct(4).GetUint64()); dsLock.SetSchemeShard(value.GetStruct(5).GetUint64()); + dsLock.SetHasWrites(value.GetStruct(6).GetBool()); return dsLock; } diff --git a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp index ba8c123a6f..6e8103fc2c 100644 --- a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp @@ -401,7 +401,8 @@ private: } if (shardIds) { LOG_D("Start resolving tablets nodes... (" << shardIds.size() << ")"); - auto kqpShardsResolver = CreateKqpShardsResolver(this->SelfId(), TxId, std::move(shardIds)); + auto kqpShardsResolver = CreateKqpShardsResolver( + this->SelfId(), TxId, false, std::move(shardIds)); KqpShardsResolverId = this->RegisterWithSameMailbox(kqpShardsResolver); } else { GetResourcesSnapshot(); diff --git a/ydb/core/kqp/executer_actor/kqp_shards_resolver.cpp b/ydb/core/kqp/executer_actor/kqp_shards_resolver.cpp index cbc586e4ad..cf9ac91382 100644 --- a/ydb/core/kqp/executer_actor/kqp_shards_resolver.cpp +++ b/ydb/core/kqp/executer_actor/kqp_shards_resolver.cpp @@ -36,11 +36,12 @@ public: } public: - TKqpShardsResolver(const TActorId& owner, ui64 txId, TSet<ui64>&& shardIds) + TKqpShardsResolver(const TActorId& owner, ui64 txId, bool useFollowers, TSet<ui64>&& shardIds) : Owner(owner) , TxId(txId) , ShardIds(std::move(shardIds)) - , TabletResolver(MakePipePeNodeCacheID(false)) + , UseFollowers(useFollowers) + , TabletResolver(MakePipePeNodeCacheID(UseFollowers)) {} void Bootstrap() { @@ -112,6 +113,7 @@ private: const TActorId Owner; const ui64 TxId; const TSet<ui64> ShardIds; + const bool UseFollowers; const TActorId TabletResolver; TMap<ui64, ui32> RetryCount; TMap<ui64, ui64> Result; @@ -119,8 +121,8 @@ private: } // anonymous namespace -IActor* CreateKqpShardsResolver(const TActorId& owner, ui64 txId, TSet<ui64>&& shardIds) { - return new TKqpShardsResolver(owner, txId, std::move(shardIds)); +IActor* CreateKqpShardsResolver(const TActorId& owner, ui64 txId, bool useFollowers, TSet<ui64>&& shardIds) { + return new TKqpShardsResolver(owner, txId, useFollowers, std::move(shardIds)); } } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/executer_actor/kqp_shards_resolver.h b/ydb/core/kqp/executer_actor/kqp_shards_resolver.h index 720a6086ab..cb0df8421a 100644 --- a/ydb/core/kqp/executer_actor/kqp_shards_resolver.h +++ b/ydb/core/kqp/executer_actor/kqp_shards_resolver.h @@ -5,6 +5,6 @@ namespace NKikimr::NKqp { -NActors::IActor* CreateKqpShardsResolver(const NActors::TActorId& owner, ui64 txId, TSet<ui64>&& shardIds); +NActors::IActor* CreateKqpShardsResolver(const NActors::TActorId& owner, ui64 txId, bool useFollowers, TSet<ui64>&& shardIds); } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/gateway/kqp_gateway.h b/ydb/core/kqp/gateway/kqp_gateway.h index 53c243a84f..8e10947c90 100644 --- a/ydb/core/kqp/gateway/kqp_gateway.h +++ b/ydb/core/kqp/gateway/kqp_gateway.h @@ -130,6 +130,7 @@ public: NKikimrKqp::EIsolationLevel IsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED; TMaybe<NKikimrKqp::TRlPath> RlPath; bool NeedTxId = true; + bool UseImmediateEffects = false; NLWTrace::TOrbit Orbit; NWilson::TTraceId TraceId; @@ -149,7 +150,8 @@ public: public: /* Compute */ - virtual NThreading::TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, TQueryData::TPtr params) = 0; + virtual NThreading::TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, + TQueryData::TPtr params, ui32 txIndex) = 0; /* Scripting */ virtual NThreading::TFuture<TQueryResult> ExplainDataQueryAst(const TString& cluster, const TString& query) = 0; diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index b8854a15d3..290d874206 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -485,8 +485,9 @@ public: } TKqpExecPureRequestHandler(IKqpGateway::TExecPhysicalRequest&& request, - TKqpRequestCounters::TPtr counters, TPromise<TResult> promise, TQueryData::TPtr params) + TKqpRequestCounters::TPtr counters, TPromise<TResult> promise, TQueryData::TPtr params, ui32 txIndex) : Request(std::move(request)) + , TxIndex(txIndex) , Parameters(params) , Counters(counters) , Promise(promise) @@ -527,7 +528,10 @@ private: result.Results.emplace_back(std::move(tx.GetMkql())); } Parameters->AddTxHolders(std::move(ev->GetTxHolders())); - Parameters->AddTxResults(std::move(txResults)); + + if (!txResults.empty()) { + Parameters->AddTxResults(TxIndex, std::move(txResults)); + } } Promise.SetValue(std::move(result)); this->PassAway(); @@ -535,6 +539,7 @@ private: private: IKqpGateway::TExecPhysicalRequest Request; + const ui32 TxIndex; TQueryData::TPtr Parameters; TKqpRequestCounters::TPtr Counters; TPromise<TResult> Promise; @@ -1008,7 +1013,7 @@ public: } } - TFuture<TGenericResult> AlterTable(Ydb::Table::AlterTableRequest&& req, const TString& cluster) override { + TFuture<TGenericResult> AlterTable(const TString& cluster, Ydb::Table::AlterTableRequest&& req, const TMaybe<TString>& requestType) override { try { if (!CheckCluster(cluster)) { return InvalidCluster<TGenericResult>(cluster); @@ -1019,7 +1024,7 @@ public: using TEvAlterTableRequest = TGrpcRequestOperationCall<Ydb::Table::AlterTableRequest, Ydb::Table::AlterTableResponse>; - return SendLocalRpcRequestNoResult<TEvAlterTableRequest>(std::move(req), Database, GetTokenCompat()); + return SendLocalRpcRequestNoResult<TEvAlterTableRequest>(std::move(req), Database, GetTokenCompat(), requestType); } catch (yexception& e) { return MakeFuture(ResultFromException<TGenericResult>(e)); @@ -1650,7 +1655,7 @@ public: } } - TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, TQueryData::TPtr params) override { + TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, TQueryData::TPtr params, ui32 txIndex) override { YQL_ENSURE(!request.Transactions.empty()); YQL_ENSURE(request.DataShardLocks.empty()); YQL_ENSURE(!request.NeedTxId); @@ -1673,7 +1678,7 @@ public: YQL_ENSURE(containOnlyPureStages(request)); auto promise = NewPromise<TExecPhysicalResult>(); - IActor* requestHandler = new TKqpExecPureRequestHandler(std::move(request), Counters, promise, params); + IActor* requestHandler = new TKqpExecPureRequestHandler(std::move(request), Counters, promise, params, txIndex); RegisterActor(requestHandler); return promise.GetFuture(); } @@ -1965,8 +1970,8 @@ private: } template<typename TRpc> - TFuture<TGenericResult> SendLocalRpcRequestNoResult(typename TRpc::TRequest&& proto, const TString& databse, const TString& token) { - return NRpcService::DoLocalRpc<TRpc>(std::move(proto), databse, token, ActorSystem).Apply([](NThreading::TFuture<typename TRpc::TResponse> future) { + TFuture<TGenericResult> SendLocalRpcRequestNoResult(typename TRpc::TRequest&& proto, const TString& databse, const TString& token, const TMaybe<TString>& requestType = {}) { + return NRpcService::DoLocalRpc<TRpc>(std::move(proto), databse, token, requestType, ActorSystem).Apply([](NThreading::TFuture<typename TRpc::TResponse> future) { auto r = future.ExtractValue(); NYql::TIssues issues; NYql::IssuesFromMessage(r.operation().issues(), issues); diff --git a/ydb/core/kqp/gateway/kqp_query_data.cpp b/ydb/core/kqp/gateway/kqp_query_data.cpp index d16c3eabc8..f3c5b19589 100644 --- a/ydb/core/kqp/gateway/kqp_query_data.cpp +++ b/ydb/core/kqp/gateway/kqp_query_data.cpp @@ -131,8 +131,8 @@ TQueryData::TQueryData(TTxAllocatorState::TPtr allocatorState) TQueryData::~TQueryData() { { auto g = TypeEnv().BindAllocator(); - TVector<TVector<TKqpExecuterTxResult>> emptyVector; - TxResults.swap(emptyVector); + THashMap<ui32, TVector<TKqpExecuterTxResult>> emptyResultMap; + TxResults.swap(emptyResultMap); TUnboxedParamsMap emptyMap; UnboxedData.swap(emptyMap); } @@ -164,8 +164,8 @@ NKikimrMiniKQL::TResult* TQueryData::GetMkqlTxResult(ui32 txIndex, ui32 resultIn return TxResults[txIndex][resultIndex].GetMkql(arena); } -void TQueryData::AddTxResults(TVector<TKqpExecuterTxResult>&& results) { - TxResults.emplace_back(std::move(results)); +void TQueryData::AddTxResults(ui32 txIndex, TVector<TKqpExecuterTxResult>&& results) { + TxResults.emplace(std::make_pair(txIndex, std::move(results))); } void TQueryData::AddTxHolders(TVector<TKqpPhyTxHolder::TConstPtr>&& holders) { @@ -280,8 +280,8 @@ void TQueryData::Clear() { Params.clear(); TUnboxedParamsMap emptyMap; UnboxedData.swap(emptyMap); - TVector<TVector<TKqpExecuterTxResult>> emptyVector; - TxResults.swap(emptyVector); + THashMap<ui32, TVector<TKqpExecuterTxResult>> emptyResultMap; + TxResults.swap(emptyResultMap); AllocState->Reset(); } } diff --git a/ydb/core/kqp/gateway/kqp_query_data.h b/ydb/core/kqp/gateway/kqp_query_data.h index ec25b7e815..d3b4e1bce6 100644 --- a/ydb/core/kqp/gateway/kqp_query_data.h +++ b/ydb/core/kqp/gateway/kqp_query_data.h @@ -160,7 +160,7 @@ private: TParamMap Params; TUnboxedParamsMap UnboxedData; - TVector<TVector<TKqpExecuterTxResult>> TxResults; + THashMap<ui32, TVector<TKqpExecuterTxResult>> TxResults; TVector<TVector<TKqpPhyTxHolder::TConstPtr>> TxHolders; TTxAllocatorState::TPtr AllocState; @@ -183,11 +183,11 @@ public: bool AddTypedValueParam(const TString& name, const Ydb::TypedValue& p); bool MaterializeParamValue(bool ensure, const NKqpProto::TKqpPhyParamBinding& paramBinding); - void AddTxResults(TVector<NKikimr::NKqp::TKqpExecuterTxResult>&& results); + void AddTxResults(ui32 txIndex, TVector<NKikimr::NKqp::TKqpExecuterTxResult>&& results); void AddTxHolders(TVector<TKqpPhyTxHolder::TConstPtr>&& holders); bool HasResult(ui32 txIndex, ui32 resultIndex) { - if (txIndex >= TxResults.size()) + if (!TxResults.contains(txIndex)) return false; return resultIndex < TxResults[txIndex].size(); diff --git a/ydb/core/kqp/host/kqp_explain_prepared.cpp b/ydb/core/kqp/host/kqp_explain_prepared.cpp index f77e6a7eed..22b65cd313 100644 --- a/ydb/core/kqp/host/kqp_explain_prepared.cpp +++ b/ydb/core/kqp/host/kqp_explain_prepared.cpp @@ -42,7 +42,7 @@ public: request.Transactions.emplace_back(tx, TransformCtx->QueryCtx->QueryData); request.NeedTxId = false; - ExecuteFuture = Gateway->ExecutePure(std::move(request), TransformCtx->QueryCtx->QueryData); + ExecuteFuture = Gateway->ExecutePure(std::move(request), TransformCtx->QueryCtx->QueryData, CurrentTxIndex); Promise = NewPromise(); ExecuteFuture.Apply([promise = Promise](const TFuture<IKqpGateway::TExecPhysicalResult> future) mutable { diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index d732ed4e2c..d8e4979926 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -970,17 +970,17 @@ public: .Build(); } - IAsyncQueryResultPtr ExecuteSchemeQuery(const TString& query, bool isSql) override { + IAsyncQueryResultPtr ExecuteSchemeQuery(const TString& query, bool isSql, const TExecSettings& settings) override { return CheckedProcessQuery(*ExprCtx, - [this, &query, isSql] (TExprContext& ctx) { - return ExecuteSchemeQueryInternal(query, isSql, ctx); + [this, &query, isSql, settings] (TExprContext& ctx) { + return ExecuteSchemeQueryInternal(query, isSql, settings, ctx); }); } - TQueryResult SyncExecuteSchemeQuery(const TString& query, bool isSql) override { + TQueryResult SyncExecuteSchemeQuery(const TString& query, bool isSql, const TExecSettings& settings) override { return CheckedSyncProcessQuery( - [this, &query, isSql] () { - return ExecuteSchemeQuery(query, isSql); + [this, &query, isSql, settings] () { + return ExecuteSchemeQuery(query, isSql, settings); }); } @@ -1133,13 +1133,13 @@ private: } settings.EndOfQueryCommit = sqlAutoCommit; settings.Flags.insert("FlexibleTypes"); + settings.Flags.insert("AnsiLike"); if (SessionCtx->Query().Type == EKikimrQueryType::Scan || SessionCtx->Query().Type == EKikimrQueryType::YqlScript || SessionCtx->Query().Type == EKikimrQueryType::YqlScriptStreaming) { - // We enable EmitAggApply and AnsiLike for filter and aggregate pushdowns to Column Shards + // We enable EmitAggApply for filter and aggregate pushdowns to Column Shards settings.Flags.insert("EmitAggApply"); - settings.Flags.insert("AnsiLike"); } else { settings.Flags.insert("DisableEmitStartsWith"); } @@ -1239,10 +1239,13 @@ private: return true; } - IAsyncQueryResultPtr ExecuteSchemeQueryInternal(const TString& query, bool isSql, TExprContext& ctx) { + IAsyncQueryResultPtr ExecuteSchemeQueryInternal(const TString& query, bool isSql, const TExecSettings& settings, TExprContext& ctx) { SetupYqlTransformer(); SessionCtx->Query().Type = EKikimrQueryType::Ddl; + if (settings.DocumentApiRestricted) { + SessionCtx->Query().DocumentApiRestricted = *settings.DocumentApiRestricted; + } TMaybe<TSqlVersion> sqlVersion; auto queryExpr = CompileYqlQuery(query, isSql, false, ctx, sqlVersion); diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h index daf6321ff1..f6ae6cd731 100644 --- a/ydb/core/kqp/host/kqp_host.h +++ b/ydb/core/kqp/host/kqp_host.h @@ -16,10 +16,16 @@ public: using IAsyncGenericResult = NYql::IKikimrAsyncResult<TGenericResult>; using IAsyncGenericResultPtr = TIntrusivePtr<IAsyncGenericResult>; - struct TPrepareSettings { + struct TExecSettings { TMaybe<bool> DocumentApiRestricted; TString ToString() const { + return TStringBuilder() << "TExecSettings{ DocumentApiRestricted: " << DocumentApiRestricted << " }"; + } + }; + + struct TPrepareSettings: public TExecSettings { + TString ToString() const { return TStringBuilder() << "TPrepareSettings{ DocumentApiRestricted: " << DocumentApiRestricted << " }"; } }; @@ -40,8 +46,8 @@ public: virtual TQueryResult SyncPrepareDataQuery(const TString& query, const TPrepareSettings& settings) = 0; /* Scheme queries */ - virtual IAsyncQueryResultPtr ExecuteSchemeQuery(const TString& query, bool isSql) = 0; - virtual TQueryResult SyncExecuteSchemeQuery(const TString& query, bool isSql) = 0; + virtual IAsyncQueryResultPtr ExecuteSchemeQuery(const TString& query, bool isSql, const TExecSettings& settings) = 0; + virtual TQueryResult SyncExecuteSchemeQuery(const TString& query, bool isSql, const TExecSettings& settings) = 0; /* Scan queries */ virtual IAsyncQueryResultPtr PrepareScanQuery(const TString& query, bool isSql, const TPrepareSettings& settings) = 0; diff --git a/ydb/core/kqp/host/kqp_type_ann.cpp b/ydb/core/kqp/host/kqp_type_ann.cpp index d43ba8de87..aadaa47636 100644 --- a/ydb/core/kqp/host/kqp_type_ann.cpp +++ b/ydb/core/kqp/host/kqp_type_ann.cpp @@ -38,14 +38,14 @@ bool CheckKeyTuple(const TKqlKeyTuple& tuple, const TKikimrTableDescription& tab auto expectedType = tableDesc.GetColumnType(meta->KeyColumnNames[i]); YQL_ENSURE(expectedType); - if (IsSameAnnotation(*expectedType, *actualType)) { - continue; - } + auto expectedItemType = expectedType->GetKind() == ETypeAnnotationKind::Optional ? + expectedType->Cast<TOptionalExprType>()->GetItemType() : expectedType; - if (expectedType->GetKind() == ETypeAnnotationKind::Optional) { - if (IsSameAnnotation(*expectedType->Cast<TOptionalExprType>()->GetItemType(), *actualType)) { - continue; - } + auto actualItemType = actualType->GetKind() == ETypeAnnotationKind::Optional ? + actualType->Cast<TOptionalExprType>()->GetItemType() : actualType; + + if (IsSameAnnotation(*expectedItemType, *actualItemType)) { + continue; } ctx.AddError(TIssue(ctx.GetPosition(tuple.Pos()), TStringBuilder() diff --git a/ydb/core/kqp/opt/physical/CMakeLists.darwin.txt b/ydb/core/kqp/opt/physical/CMakeLists.darwin.txt index 02f28dab6d..732a7acf47 100644 --- a/ydb/core/kqp/opt/physical/CMakeLists.darwin.txt +++ b/ydb/core/kqp/opt/physical/CMakeLists.darwin.txt @@ -28,6 +28,7 @@ target_sources(kqp-opt-physical PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_precompute.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_source.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_helpers.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_stage_float_up.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp diff --git a/ydb/core/kqp/opt/physical/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/opt/physical/CMakeLists.linux-aarch64.txt index 8e3ecba7b7..0ad95d5845 100644 --- a/ydb/core/kqp/opt/physical/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/opt/physical/CMakeLists.linux-aarch64.txt @@ -29,6 +29,7 @@ target_sources(kqp-opt-physical PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_precompute.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_source.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_helpers.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_stage_float_up.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp diff --git a/ydb/core/kqp/opt/physical/CMakeLists.linux.txt b/ydb/core/kqp/opt/physical/CMakeLists.linux.txt index 8e3ecba7b7..0ad95d5845 100644 --- a/ydb/core/kqp/opt/physical/CMakeLists.linux.txt +++ b/ydb/core/kqp/opt/physical/CMakeLists.linux.txt @@ -29,6 +29,7 @@ target_sources(kqp-opt-physical PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_precompute.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_source.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_helpers.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy_stage_float_up.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp index edaedc05ca..28b895f741 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp @@ -32,9 +32,7 @@ public: AddHandler(0, &TKqlReadTableRanges::Match, HNDL(BuildReadTableRangesStage)); AddHandler(0, &TKqlLookupTable::Match, HNDL(BuildLookupTableStage)); AddHandler(0, &TKqlStreamLookupTable::Match, HNDL(BuildStreamLookupTableStages)); - AddHandler(0, [](const TExprNode* node) { return TCoSort::Match(node) || TCoTopSort::Match(node); }, - HNDL(RemoveRedundantSortByPk)); - AddHandler(0, &TDqStage::Match, HNDL(RemoveRedundantSortByPkOverSource)); + AddHandler(0, [](auto) { return true; }, HNDL(RemoveRedundantSortByPk)); AddHandler(0, &TCoTake::Match, HNDL(ApplyLimitToReadTable)); AddHandler(0, &TCoFlatMap::Match, HNDL(PushOlapFilter)); AddHandler(0, &TCoAggregateCombine::Match, HNDL(PushAggregateCombineToStage)); @@ -77,7 +75,6 @@ public: AddHandler(0, &TCoAsList::Match, HNDL(PropagatePrecomuteScalarRowset<false>)); AddHandler(0, &TCoTake::Match, HNDL(PropagatePrecomuteTake<false>)); AddHandler(0, &TCoFlatMap::Match, HNDL(PropagatePrecomuteFlatmap<false>)); - AddHandler(0, &TDqStage::Match, HNDL(ApplyLimitToReadTableSource)); AddHandler(0, &TCoAggregateCombine::Match, HNDL(ExpandAggregatePhase)); AddHandler(0, &TCoAggregateCombineState::Match, HNDL(ExpandAggregatePhase)); @@ -106,7 +103,7 @@ public: AddHandler(1, &TCoTake::Match, HNDL(PropagatePrecomuteTake<true>)); AddHandler(1, &TCoFlatMap::Match, HNDL(PropagatePrecomuteFlatmap<true>)); - AddHandler(2, &TDqStage::Match, HNDL(ExpandNullMembersForReadTableSource)); + AddHandler(2, &TDqStage::Match, HNDL(RewriteKqpReadTable)); #undef HNDL SetGlobal(1u); @@ -138,27 +135,15 @@ protected: return output; } - TMaybeNode<TExprBase> RemoveRedundantSortByPkOverSource(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpRemoveRedundantSortByPkOverSource(node, ctx, KqpCtx); - DumpAppliedRule("RemoveRedundantSortByPkOverSource", node.Ptr(), output.Ptr(), ctx); - return output; - } - TMaybeNode<TExprBase> RemoveRedundantSortByPk(TExprBase node, TExprContext& ctx) { TExprBase output = KqpRemoveRedundantSortByPk(node, ctx, KqpCtx); DumpAppliedRule("RemoveRedundantSortByPk", node.Ptr(), output.Ptr(), ctx); return output; } - TMaybeNode<TExprBase> ExpandNullMembersForReadTableSource(TExprBase node, TExprContext& ctx) { - TExprBase output = ExpandSkipNullMembersForReadTableSource(node, ctx, KqpCtx); - DumpAppliedRule("ExpandSkipNullMembersForReadTableSource", node.Ptr(), output.Ptr(), ctx); - return output; - } - - TMaybeNode<TExprBase> ApplyLimitToReadTableSource(TExprBase node, TExprContext& ctx) { - TExprBase output = KqpApplyLimitToReadTableSource(node, ctx, KqpCtx); - DumpAppliedRule("ApplyLimitToReadTableSource", node.Ptr(), output.Ptr(), ctx); + TMaybeNode<TExprBase> RewriteKqpReadTable(TExprBase node, TExprContext& ctx) { + TExprBase output = KqpRewriteReadTable(node, ctx, KqpCtx); + DumpAppliedRule("RewriteKqpReadTable", node.Ptr(), output.Ptr(), ctx); return output; } diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp index 193ed8d04b..2b75de04a7 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp @@ -54,72 +54,6 @@ TMaybeNode<TDqPhyPrecompute> BuildLookupKeysPrecompute(const TExprBase& input, T .Done(); } -// ReadRangesSource can't deal with skipnullkeys, so we should expand it to (ExtractMembers (SkipNullKeys)) -//FIXME: simplify KIKIMR-16987 -NYql::NNodes::TExprBase ExpandSkipNullMembersForReadTableSource(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext&) { - auto stage = node.Cast<TDqStage>(); - TMaybe<size_t> tableSourceIndex; - for (size_t i = 0; i < stage.Inputs().Size(); ++i) { - auto input = stage.Inputs().Item(i); - if (input.Maybe<TDqSource>() && input.Cast<TDqSource>().Settings().Maybe<TKqpReadRangesSourceSettings>()) { - tableSourceIndex = i; - } - } - if (!tableSourceIndex) { - return node; - } - - auto source = stage.Inputs().Item(*tableSourceIndex).Cast<TDqSource>(); - auto readRangesSource = source.Settings().Cast<TKqpReadRangesSourceSettings>(); - auto settings = TKqpReadTableSettings::Parse(readRangesSource.Settings()); - - if (settings.SkipNullKeys.empty()) { - return node; - } - - auto sourceArg = stage.Program().Args().Arg(*tableSourceIndex); - - THashSet<TString> seenColumns; - TVector<TCoAtom> columns; - TVector<TCoAtom> skipNullColumns; - for (size_t i = 0; i < readRangesSource.Columns().Size(); ++i) { - auto atom = readRangesSource.Columns().Item(i); - auto column = atom.StringValue(); - if (seenColumns.insert(column).second) { - columns.push_back(atom); - } - } - for (auto& column : settings.SkipNullKeys) { - TCoAtom atom(ctx.NewAtom(readRangesSource.Settings().Pos(), column)); - skipNullColumns.push_back(atom); - if (seenColumns.insert(column).second) { - columns.push_back(atom); - } - } - - settings.SkipNullKeys.clear(); - auto newSettings = Build<TKqpReadRangesSourceSettings>(ctx, source.Pos()) - .Table(readRangesSource.Table()) - .Columns().Add(columns).Build() - .Settings(settings.BuildNode(ctx, source.Settings().Pos())) - .RangesExpr(readRangesSource.RangesExpr()) - .ExplainPrompt(readRangesSource.ExplainPrompt()) - .Done(); - TDqStage replacedSettings = ReplaceTableSourceSettings(stage, *tableSourceIndex, newSettings, ctx); - - TCoArgument replaceArg{ctx.NewArgument(sourceArg.Pos(), TStringBuilder() << "_kqp_source_arg_0")}; - auto replaceExpr = - Build<TCoExtractMembers>(ctx, node.Pos()) - .Members(readRangesSource.Columns()) - .Input<TCoSkipNullMembers>() - .Input(replaceArg) - .Members().Add(skipNullColumns).Build() - .Build() - .Done(); - - return ReplaceStageArg(replacedSettings, *tableSourceIndex, replaceArg, replaceExpr, ctx); -} - TExprBase KqpBuildReadTableStage(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { if (!node.Maybe<TKqlReadTable>()) { return node; @@ -127,12 +61,6 @@ TExprBase KqpBuildReadTableStage(TExprBase node, TExprContext& ctx, const TKqpOp const TKqlReadTable& read = node.Cast<TKqlReadTable>(); auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); - bool useSource = kqpCtx.Config->EnableKqpScanQuerySourceRead && kqpCtx.IsScanQuery(); - useSource = useSource || (kqpCtx.Config->EnableKqpDataQuerySourceRead && kqpCtx.IsDataQuery()); - useSource = useSource && - tableDesc.Metadata->Kind != EKikimrTableKind::SysView && - tableDesc.Metadata->Kind != EKikimrTableKind::Olap; - TVector<TExprBase> values; TNodeOnNodeOwnedMap replaceMap; @@ -198,58 +126,24 @@ TExprBase KqpBuildReadTableStage(TExprBase node, TExprContext& ctx, const TKqpOp .Build() .Done(); - if (useSource) { - for (size_t i = 0; i < values.size(); ++i) { - auto replace = Build<TCoNth>(ctx, read.Pos()) - .Tuple(precompute) - .Index().Build(ToString(i)) - .Done() - .Ptr(); - - rangeReplaces[values[i].Raw()] = replace; - } - } else { - TCoArgument arg{ctx.NewArgument(read.Pos(), TStringBuilder() << "_kqp_pc_arg_0")}; - programArgs.push_back(arg); + TCoArgument arg{ctx.NewArgument(read.Pos(), TStringBuilder() << "_kqp_pc_arg_0")}; + programArgs.push_back(arg); - for (size_t i = 0; i < values.size(); ++i) { - auto replace = Build<TCoNth>(ctx, read.Pos()) - .Tuple(arg) - .Index().Build(ToString(i)) - .Done() - .Ptr(); + for (size_t i = 0; i < values.size(); ++i) { + auto replace = Build<TCoNth>(ctx, read.Pos()) + .Tuple(arg) + .Index().Build(ToString(i)) + .Done() + .Ptr(); - rangeReplaces[values[i].Raw()] = replace; - } - inputs.push_back(precompute); + rangeReplaces[values[i].Raw()] = replace; } - } - - if (useSource) { - inputs.push_back( - Build<TDqSource>(ctx, read.Pos()) - .Settings<TKqpReadRangesSourceSettings>() - .Table(read.Table()) - .Columns(read.Columns()) - .Settings(read.Settings()) - .RangesExpr(ctx.ReplaceNodes(read.Range().Ptr(), rangeReplaces)) - .Build() - .DataSource<TCoDataSource>() - .Category<TCoAtom>().Value(KqpReadRangesSourceName).Build() - .Build() - .Done()); + inputs.push_back(precompute); } TMaybeNode<TExprBase> phyRead; switch (tableDesc.Metadata->Kind) { case EKikimrTableKind::Datashard: - if (useSource) { - TCoArgument arg{ctx.NewArgument(read.Pos(), TStringBuilder() << "_kqp_source_arg")}; - programArgs.push_back(arg); - - phyRead = arg; - break; - } case EKikimrTableKind::SysView: phyRead = Build<TKqpReadTable>(ctx, read.Pos()) .Table(read.Table()) @@ -283,7 +177,6 @@ TExprBase KqpBuildReadTableStage(TExprBase node, TExprContext& ctx, const TKqpOp .Done(); } - TExprBase KqpBuildReadTableRangesStage(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, const TParentsMap& parents) { @@ -295,12 +188,6 @@ TExprBase KqpBuildReadTableRangesStage(TExprBase node, TExprContext& ctx, auto ranges = read.Ranges(); auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); - bool useSource = kqpCtx.Config->EnableKqpScanQuerySourceRead && kqpCtx.IsScanQuery(); - useSource = useSource || (kqpCtx.Config->EnableKqpDataQuerySourceRead && kqpCtx.IsDataQuery()); - useSource = useSource && - tableDesc.Metadata->Kind != EKikimrTableKind::SysView && - tableDesc.Metadata->Kind != EKikimrTableKind::Olap; - bool fullScan = TCoVoid::Match(ranges.Raw()); TVector<TExprBase> input; @@ -387,58 +274,28 @@ TExprBase KqpBuildReadTableRangesStage(TExprBase node, TExprContext& ctx, .Done(); rangesExpr = precompute; - if (!useSource) { - argument = Build<TCoArgument>(ctx, read.Pos()) - .Name("_kqp_pc_ranges_arg_0") - .Done(); + argument = Build<TCoArgument>(ctx, read.Pos()) + .Name("_kqp_pc_ranges_arg_0") + .Done(); - input.push_back(precompute); - programArgs.push_back(argument.Cast<TCoArgument>()); - } + input.push_back(precompute); + programArgs.push_back(argument.Cast<TCoArgument>()); } else { rangesExpr = argument = read.Ranges(); } TMaybeNode<TExprBase> phyRead; - TMaybeNode<TExprBase> sourceArg; - if (useSource) { - YQL_ENSURE(rangesExpr.IsValid()); - - input.push_back( - Build<TDqSource>(ctx, read.Pos()) - .Settings<TKqpReadRangesSourceSettings>() - .Table(read.Table()) - .Columns(read.Columns()) - .Settings(read.Settings()) - .RangesExpr(rangesExpr.Cast()) - .ExplainPrompt(read.ExplainPrompt()) - .Build() - .DataSource<TCoDataSource>() - .Category<TCoAtom>().Value(KqpReadRangesSourceName).Build() - .Build() - .Done()); - sourceArg = Build<TCoArgument>(ctx, read.Pos()) - .Name("_kqp_pc_source_arg_0") - .Done(); - programArgs.push_back(sourceArg.Cast<TCoArgument>()); - } - switch (tableDesc.Metadata->Kind) { case EKikimrTableKind::Datashard: case EKikimrTableKind::SysView: - if (useSource) { - YQL_ENSURE(sourceArg.IsValid()); - phyRead = sourceArg.Cast(); - } else { - phyRead = Build<TKqpReadTableRanges>(ctx, read.Pos()) - .Table(read.Table()) - .Ranges(argument.Cast()) - .Columns(read.Columns()) - .Settings(read.Settings()) - .ExplainPrompt(read.ExplainPrompt()) - .Done(); - } + phyRead = Build<TKqpReadTableRanges>(ctx, read.Pos()) + .Table(read.Table()) + .Ranges(argument.Cast()) + .Columns(read.Columns()) + .Settings(read.Settings()) + .ExplainPrompt(read.ExplainPrompt()) + .Done(); break; case EKikimrTableKind::Olap: diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_limit.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_limit.cpp index a32ef61a33..e538b09367 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_limit.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_limit.cpp @@ -8,164 +8,6 @@ namespace NKikimr::NKqp::NOpt { using namespace NYql; using namespace NYql::NNodes; -THashSet<const TExprNode*> CollectConnections(TDqStage stage, TExprBase node) { - THashSet<const TExprNode*> args; - for (auto&& arg : stage.Program().Args()) { - args.insert(arg.Raw()); - } - - THashSet<const TExprNode*> result; - TNodeOnNodeOwnedMap replaceMap; - VisitExpr(node.Ptr(), - [&](const TExprNode::TPtr& exprPtr) -> bool { - TExprBase expr(exprPtr); - if (expr.Maybe<TDqConnection>()) { - return false; - } - if (args.contains(exprPtr.Get())) { - result.insert(exprPtr.Get()); - } - return true; - }); - return result; -} - -//FIXME: simplify KIKIMR-16987 -TExprBase KqpApplyLimitToReadTableSource(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext&) { - auto stage = node.Cast<TDqStage>(); - TMaybe<size_t> tableSourceIndex; - for (size_t i = 0; i < stage.Inputs().Size(); ++i) { - auto input = stage.Inputs().Item(i); - if (input.Maybe<TDqSource>() && input.Cast<TDqSource>().Settings().Maybe<TKqpReadRangesSourceSettings>()) { - tableSourceIndex = i; - } - } - if (!tableSourceIndex) { - return node; - } - - auto source = stage.Inputs().Item(*tableSourceIndex).Cast<TDqSource>(); - auto readRangesSource = source.Settings().Cast<TKqpReadRangesSourceSettings>(); - auto settings = TKqpReadTableSettings::Parse(readRangesSource.Settings()); - - if (settings.ItemsLimit) { - return node; // already set? - } - - auto sourceArg = stage.Program().Args().Arg(*tableSourceIndex); - TExprNode::TPtr foundTake; - bool singleConsumer = true; - VisitExpr(stage.Program().Body().Ptr(), - [&](const TExprNode::TPtr& exprPtr) -> bool { - TExprBase expr(exprPtr); - if (expr.Maybe<TDqConnection>() || expr.Maybe<TDqPrecompute>() || expr.Maybe<TDqPhyPrecompute>()) { - return false; - } - if (auto take = expr.Maybe<TCoTake>()) { - auto maybeSkip = take.Input().Maybe<TCoSkip>(); - auto input = (maybeSkip ? maybeSkip.Cast().Input() : take.Input()).Cast(); - if (input.Raw() == sourceArg.Raw()) { - auto ptr = take.Cast().Ptr(); - if (foundTake && foundTake != ptr) { - singleConsumer = false; - } - foundTake = ptr; - } - } - return true; - }); - - if (!singleConsumer || !foundTake) { - return node; - } - - auto take = TCoTake(foundTake); - - auto maybeSkip = take.Input().Maybe<TCoSkip>(); - auto input = maybeSkip ? maybeSkip.Cast().Input() : take.Input(); - - TMaybeNode<TExprBase> limitValue; - auto maybeTakeCount = take.Count().Maybe<TCoUint64>(); - auto maybeSkipCount = maybeSkip.Count().Maybe<TCoUint64>(); - - if (maybeTakeCount && (!maybeSkip || maybeSkipCount)) { - ui64 totalLimit = FromString<ui64>(maybeTakeCount.Cast().Literal().Value()); - - if (maybeSkipCount) { - totalLimit += FromString<ui64>(maybeSkipCount.Cast().Literal().Value()); - } - - limitValue = Build<TCoUint64>(ctx, node.Pos()) - .Literal<TCoAtom>() - .Value(ToString(totalLimit)).Build() - .Done(); - } else { - limitValue = take.Count(); - if (maybeSkip) { - limitValue = Build<TCoPlus>(ctx, node.Pos()) - .Left(limitValue.Cast()) - .Right(maybeSkip.Cast().Count()) - .Done(); - } - } - - YQL_CLOG(TRACE, ProviderKqp) << "-- set limit items value to " << limitValue.Cast().Ref().Dump(); - - if (limitValue.Maybe<TCoUint64>()) { - settings.SetItemsLimit(limitValue.Cast().Ptr()); - } else { - if (auto args = CollectConnections(stage, limitValue.Cast())) { - TVector<TCoArgument> stageArgs; - TVector<TExprBase> inputs; - TNodeOnNodeOwnedMap replaces; - - size_t index = 0; - for (auto&& arg : stage.Program().Args()) { - if (args.contains(arg.Raw())) { - TCoArgument replace{ctx.NewArgument(node.Pos(), TStringBuilder() << "_kqp_pc_arg_" << index)}; - inputs.push_back(stage.Inputs().Item(index)); - stageArgs.push_back(replace); - replaces[arg.Raw()] = replace.Ptr(); - } - index += 1; - } - - limitValue = Build<TDqCnValue>(ctx, node.Pos()) - .Output() - .Stage<TDqStage>() - .Settings().Build() - .Inputs().Add(inputs).Build() - .Program<TCoLambda>() - .Args(stageArgs) - .Body<TCoToStream>() - .Input<TCoJust>() - .Input(ctx.ReplaceNodes(limitValue.Cast().Ptr(), replaces)) - .Build() - .Build() - .Build() - .Build() - .Index().Build("0") - .Build() - .Done(); - } - - settings.SetItemsLimit(Build<TDqPrecompute>(ctx, node.Pos()) - .Input(limitValue.Cast()) - .Done().Ptr()); - } - - auto newSettings = Build<TKqpReadRangesSourceSettings>(ctx, source.Pos()) - .Table(readRangesSource.Table()) - .Columns(readRangesSource.Columns()) - .Settings(settings.BuildNode(ctx, source.Pos())) - .RangesExpr(readRangesSource.RangesExpr()) - .ExplainPrompt(readRangesSource.ExplainPrompt()) - .Done(); - - return ReplaceTableSourceSettings(stage, *tableSourceIndex, newSettings, ctx); -} - - TExprBase KqpApplyLimitToReadTable(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { if (!node.Maybe<TCoTake>()) { return node; diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter_collection.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter_collection.cpp index 2719be8424..b9312668a0 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter_collection.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter_collection.cpp @@ -13,6 +13,33 @@ using namespace NYql::NNodes; namespace { +bool ColumnHasBinaryStringType(const TExprBase& expr) { + if (!expr.Maybe<TCoMember>()) { + return false; + } + auto typeAnn = expr.Ptr()->GetTypeAnn(); + auto itemType = GetSeqItemType(typeAnn); + if (!itemType) { + itemType = typeAnn; + } + if (itemType->GetKind() != ETypeAnnotationKind::Data) { + return false; + } + auto dataTypeInfo = NUdf::GetDataTypeInfo(itemType->Cast<TDataExprType>()->GetSlot()); + return (std::string(dataTypeInfo.Name.data()) == "String"); +} + +bool IsLikeOperator(const TCoCompare& predicate) { + if (predicate.Maybe<TCoCmpStringContains>()) { + return true; + } else if (predicate.Maybe<TCoCmpStartsWith>()) { + return true; + } else if (predicate.Maybe<TCoCmpEndsWith>()) { + return true; + } + return false; +} + bool IsSupportedPredicate(const TCoCompare& predicate) { if (predicate.Maybe<TCoCmpEqual>()) { return true; @@ -30,13 +57,7 @@ bool IsSupportedPredicate(const TCoCompare& predicate) { return true; } else if (NKikimr::NSsa::RuntimeVersion >= 2U) { // We introduced LIKE pushdown in v2 of SSA program - if (predicate.Maybe<TCoCmpStringContains>()) { - return true; - } else if (predicate.Maybe<TCoCmpStartsWith>()) { - return true; - } else if (predicate.Maybe<TCoCmpEndsWith>()) { - return true; - } + return IsLikeOperator(predicate); } return false; @@ -273,6 +294,10 @@ bool CheckComparisonParametersForPushdown(const TCoCompare& compare, const TExpr if (!IsComparableTypes(leftList[i], rightList[i], equality, inputType)) { return false; } + if (IsLikeOperator(compare) && (ColumnHasBinaryStringType(leftList[i]) || ColumnHasBinaryStringType(rightList[i]))) { + // Currently Column Shard doesn't have LIKE kernel for binary strings + return false; + } } return true; diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h b/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h index 512251ddd9..98fb1e5732 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h @@ -11,6 +11,8 @@ namespace NKikimr::NKqp::NOpt { +NYql::NNodes::TExprBase KqpRewriteReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); + NYql::NNodes::TExprBase KqpBuildReadTableStage(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); @@ -24,16 +26,9 @@ NYql::NNodes::TExprBase KqpBuildStreamLookupTableStages(NYql::NNodes::TExprBase NYql::NNodes::TExprBase KqpRemoveRedundantSortByPk(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); -NYql::NNodes::TExprBase KqpRemoveRedundantSortByPkOverSource(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, - const TKqpOptimizeContext& kqpCtx); - -NYql::NNodes::TExprBase KqpApplyLimitToReadTableSource(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); - NYql::NNodes::TExprBase KqpApplyLimitToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); -NYql::NNodes::TExprBase ExpandSkipNullMembersForReadTableSource(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); - NYql::NNodes::TExprBase KqpPushOlapFilter(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, NYql::TTypeAnnotationContext& typesCtx); @@ -50,4 +45,6 @@ NYql::NNodes::TExprBase KqpPropagatePrecomuteScalarRowset(NYql::NNodes::TExprBas bool AllowFuseJoinInputs(NYql::NNodes::TExprBase node); +bool UseSource(const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrTableDescription& tableDesc); + } // NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp index 4b09bf0655..686757d9ed 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp @@ -16,14 +16,7 @@ using namespace NYql::NNodes; using TTableData = std::pair<const NYql::TKikimrTableDescription*, NYql::TKqpReadTableSettings>; -TExprBase KqpRemoveRedundantSortByPkBase( - TExprBase node, - TExprContext& ctx, - const TKqpOptimizeContext& kqpCtx, - std::function<TMaybe<TTableData>(TExprBase)> tableAccessor, - std::function<TExprBase(TExprBase, NYql::TKqpReadTableSettings)> rebuildInput, - bool allowSortForAllTables = false) -{ +TExprBase KqpRemoveRedundantSortByPk(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { auto maybeSort = node.Maybe<TCoSort>(); auto maybeTopSort = node.Maybe<TCoTopSort>(); @@ -83,12 +76,13 @@ TExprBase KqpRemoveRedundantSortByPkBase( } } - auto tableData = tableAccessor(input); - if (!tableData) { + bool isReadTable = input.Maybe<TKqpReadTable>().IsValid(); + bool isReadTableRanges = input.Maybe<TKqpReadTableRanges>().IsValid() || input.Maybe<TKqpReadOlapTableRanges>().IsValid() ; + if (!isReadTable && !isReadTableRanges) { return node; } - auto& tableDesc = *tableData->first; - auto settings = tableData->second; + auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, GetReadTablePath(input, isReadTableRanges)); + auto settings = GetReadTableSettings(input, isReadTableRanges); auto checkKey = [keySelector, &tableDesc, &passthroughFields] (TExprBase key, ui32 index) { if (!key.Maybe<TCoMember>()) { @@ -129,7 +123,7 @@ TExprBase KqpRemoveRedundantSortByPkBase( bool olapTable = tableDesc.Metadata->Kind == EKikimrTableKind::Olap; if (direction == SortDirectionReverse) { - if (!allowSortForAllTables && !olapTable && kqpCtx.IsScanQuery()) { + if (!UseSource(kqpCtx, tableDesc) && !olapTable && kqpCtx.IsScanQuery()) { return node; } @@ -140,11 +134,11 @@ TExprBase KqpRemoveRedundantSortByPkBase( settings.SetReverse(); settings.SetSorted(); - input = rebuildInput(input, settings); + input = BuildReadNode(input.Pos(), ctx, input, settings); } else if (direction == SortDirectionForward) { - if (olapTable || allowSortForAllTables) { + if (olapTable || UseSource(kqpCtx, tableDesc)) { settings.SetSorted(); - input = rebuildInput(input, settings); + input = BuildReadNode(input.Pos(), ctx, input, settings); } } @@ -165,103 +159,5 @@ TExprBase KqpRemoveRedundantSortByPkBase( } } -TExprBase KqpRemoveRedundantSortByPk(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - return KqpRemoveRedundantSortByPkBase(node, ctx, kqpCtx, - [&](TExprBase input) -> TMaybe<TTableData> { - bool isReadTable = input.Maybe<TKqpReadTable>().IsValid(); - bool isReadTableRanges = input.Maybe<TKqpReadTableRanges>().IsValid() || input.Maybe<TKqpReadOlapTableRanges>().IsValid() ; - if (!isReadTable && !isReadTableRanges) { - return Nothing(); - } - auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, GetReadTablePath(input, isReadTableRanges)); - auto settings = GetReadTableSettings(input, isReadTableRanges); - return TTableData{&tableDesc, settings}; - }, - [&](TExprBase input, NYql::TKqpReadTableSettings settings) { - return BuildReadNode(input.Pos(), ctx, input, settings); - }); -} - -//FIXME: simplify KIKIMR-16987 -NYql::NNodes::TExprBase KqpRemoveRedundantSortByPkOverSource( - NYql::NNodes::TExprBase node, NYql::TExprContext& exprCtx, const TKqpOptimizeContext& kqpCtx) -{ - auto stage = node.Cast<TDqStage>(); - TMaybe<size_t> tableSourceIndex; - for (size_t i = 0; i < stage.Inputs().Size(); ++i) { - auto input = stage.Inputs().Item(i); - if (input.Maybe<TDqSource>() && input.Cast<TDqSource>().Settings().Maybe<TKqpReadRangesSourceSettings>()) { - tableSourceIndex = i; - } - } - if (!tableSourceIndex) { - return node; - } - - auto source = stage.Inputs().Item(*tableSourceIndex).Cast<TDqSource>(); - auto readRangesSource = source.Settings().Cast<TKqpReadRangesSourceSettings>(); - auto settings = TKqpReadTableSettings::Parse(readRangesSource.Settings()); - - auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, readRangesSource.Table().Path()); - - TVector<NYql::TKqpReadTableSettings> newSettings; - NYql::TNodeOnNodeOwnedMap bodyReplaces; - VisitExpr(stage.Program().Body().Ptr(), - [&](const TExprNode::TPtr& exprPtr) -> bool { - TExprBase expr(exprPtr); - if (expr.Maybe<TDqConnection>() || expr.Maybe<TDqPrecompute>() || expr.Maybe<TDqPhyPrecompute>()) { - return false; - } - if (TCoSort::Match(expr.Raw()) || TCoTopSort::Match(expr.Raw())) { - auto newExpr = KqpRemoveRedundantSortByPkBase(expr, exprCtx, kqpCtx, - [&](TExprBase node) -> TMaybe<TTableData> { - if (node.Ptr() != node.Ptr()) { - return Nothing(); - } - return TTableData{&tableDesc, settings}; - }, - [&](TExprBase input, NYql::TKqpReadTableSettings settings) { - newSettings.push_back(settings); - return input; - }, - /* allowSortForAllTables */ true); - if (newExpr.Ptr() != expr.Ptr()) { - bodyReplaces[expr.Raw()] = newExpr.Ptr(); - } - } - return true; - }); - - if (newSettings) { - for (size_t i = 1; i < newSettings.size(); ++i) { - if (newSettings[0] != newSettings[i]) { - return node; - } - } - - if (settings != newSettings[0]) { - auto newSource = Build<TKqpReadRangesSourceSettings>(exprCtx, source.Pos()) - .Table(readRangesSource.Table()) - .Columns(readRangesSource.Columns()) - .Settings(newSettings[0].BuildNode(exprCtx, source.Settings().Pos())) - .RangesExpr(readRangesSource.RangesExpr()) - .ExplainPrompt(readRangesSource.ExplainPrompt()) - .Done(); - stage = ReplaceTableSourceSettings(stage, *tableSourceIndex, newSource, exprCtx); - } - } - - if (bodyReplaces.empty()) { - return stage; - } - - return Build<TDqStage>(exprCtx, stage.Pos()) - .Inputs(stage.Inputs()) - .Outputs(stage.Outputs()) - .Settings(stage.Settings()) - .Program(TCoLambda(exprCtx.ReplaceNodes(stage.Program().Ptr(), bodyReplaces))) - .Done(); -} - } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_source.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_source.cpp new file mode 100644 index 0000000000..8b9e1f81e1 --- /dev/null +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_source.cpp @@ -0,0 +1,171 @@ +#include "kqp_opt_phy_rules.h" + +#include <ydb/core/kqp/common/kqp_yql.h> +#include <ydb/core/kqp/opt/kqp_opt_impl.h> +#include <ydb/core/kqp/opt/physical/kqp_opt_phy_impl.h> +#include <ydb/core/tx/schemeshard/schemeshard_utils.h> + +#include <ydb/public/lib/scheme_types/scheme_type_id.h> + +#include <ydb/library/yql/dq/opt/dq_opt.h> +#include <ydb/library/yql/core/yql_opt_utils.h> + +namespace NKikimr::NKqp::NOpt { + +using namespace NYql; +using namespace NYql::NDq; +using namespace NYql::NNodes; + + +bool UseSource(const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrTableDescription& tableDesc) { + bool useSource = kqpCtx.Config->EnableKqpScanQuerySourceRead && kqpCtx.IsScanQuery(); + useSource = useSource || (kqpCtx.Config->EnableKqpDataQuerySourceRead && kqpCtx.IsDataQuery()); + useSource = useSource && + tableDesc.Metadata->Kind != EKikimrTableKind::SysView && + tableDesc.Metadata->Kind != EKikimrTableKind::Olap; + + return useSource; +} + +TExprBase KqpRewriteReadTable(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { + auto stage = node.Cast<TDqStage>(); + struct TMatchedRead { + TExprBase Expr; + TKqpTable Table; + TCoAtomList Columns; + TCoNameValueTupleList Settings; + TExprBase RangeExpr; + TMaybeNode<TCoNameValueTupleList> ExplainPrompt = {}; + }; + TMaybe<TMatchedRead> matched; + + TMaybeNode<TKqpReadTable> mayberead; + VisitExpr(stage.Program().Body().Ptr(), [&](const TExprNode::TPtr& node) { + TExprBase expr(node); + if (auto cast = expr.Maybe<TKqpReadTable>()) { + Y_ENSURE(!matched || matched->Expr.Raw() == node.Get()); + auto read = cast.Cast(); + matched = TMatchedRead { + .Expr = read, + .Table = read.Table(), + .Columns = read.Columns(), + .Settings = read.Settings(), + .RangeExpr = read.Range() + }; + } + + if (auto cast = expr.Maybe<TKqpReadTableRanges>()) { + Y_ENSURE(!matched || matched->Expr.Raw() == node.Get()); + auto read = cast.Cast(); + matched = TMatchedRead { + .Expr = read, + .Table = read.Table(), + .Columns = read.Columns(), + .Settings = read.Settings(), + .RangeExpr = read.Ranges(), + .ExplainPrompt = read.ExplainPrompt() + }; + } + return true; + }); + + if (!matched) { + return node; + } + + auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, matched->Table.Path()); + if (!UseSource(kqpCtx, tableDesc)) { + return node; + } + + auto settings = TKqpReadTableSettings::Parse(matched->Settings); + auto selectColumns = matched->Columns; + TVector<TCoAtom> skipNullColumns; + if (settings.SkipNullKeys) { + THashSet<TString> seenColumns; + TVector<TCoAtom> columns; + + for (size_t i = 0; i < matched->Columns.Size(); ++i) { + auto atom = matched->Columns.Item(i); + auto column = atom.StringValue(); + if (seenColumns.insert(column).second) { + columns.push_back(atom); + } + } + for (auto& column : settings.SkipNullKeys) { + TCoAtom atom(ctx.NewAtom(matched->Settings.Pos(), column)); + skipNullColumns.push_back(atom); + if (seenColumns.insert(column).second) { + columns.push_back(atom); + } + } + + matched->Columns = Build<TCoAtomList>(ctx, matched->Columns.Pos()).Add(columns).Done(); + + settings.SkipNullKeys.clear(); + matched->Settings = settings.BuildNode(ctx, matched->Settings.Pos()); + } + + TVector<TExprBase> inputs; + TVector<TCoArgument> args; + TNodeOnNodeOwnedMap argReplaces; + TNodeOnNodeOwnedMap sourceReplaces; + + for (size_t i = 0; i < stage.Inputs().Size(); ++i) { + inputs.push_back(stage.Inputs().Item(i)); + + TCoArgument newArg{ctx.NewArgument(stage.Pos(), TStringBuilder() << "_kqp_pc_arg_" << i)}; + args.push_back(newArg); + + TCoArgument arg = stage.Program().Args().Arg(i); + + argReplaces[arg.Raw()] = newArg.Ptr(); + sourceReplaces[arg.Raw()] = stage.Inputs().Item(i).Ptr(); + } + + TCoArgument arg{ctx.NewArgument(stage.Pos(), TStringBuilder() << "_kqp_source_arg")}; + args.insert(args.begin(), arg); + if (skipNullColumns) { + argReplaces[matched->Expr.Raw()] = + Build<TCoExtractMembers>(ctx, node.Pos()) + .Members(selectColumns) + .Input<TCoSkipNullMembers>() + .Input<TCoToFlow>().Input(arg).Build() + .Members().Add(skipNullColumns).Build() + .Build() + .Done().Ptr(); + } else { + argReplaces[matched->Expr.Raw()] = + Build<TCoToFlow>(ctx, matched->Expr.Pos()) + .Input(arg) + .Done() + .Ptr(); + } + + auto source = + Build<TDqSource>(ctx, matched->Expr.Pos()) + .Settings<TKqpReadRangesSourceSettings>() + .Table(matched->Table) + .Columns(matched->Columns) + .Settings(matched->Settings) + .RangesExpr(matched->RangeExpr) + .ExplainPrompt(matched->ExplainPrompt) + .Build() + .DataSource<TCoDataSource>() + .Category<TCoAtom>().Value(KqpReadRangesSourceName).Build() + .Build() + .Done(); + inputs.insert(inputs.begin(), TExprBase(ctx.ReplaceNodes(source.Ptr(), sourceReplaces))); + + return Build<TDqStage>(ctx, stage.Pos()) + .Inputs().Add(inputs).Build() + .Outputs(stage.Outputs()) + .Settings(stage.Settings()) + .Program() + .Args(args) + .Body(ctx.ReplaceNodes(stage.Program().Body().Ptr(), argReplaces)) + .Build() + .Done(); +} + +} // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/provider/CMakeLists.darwin.txt b/ydb/core/kqp/provider/CMakeLists.darwin.txt index 78868c23b2..08ae966aa0 100644 --- a/ydb/core/kqp/provider/CMakeLists.darwin.txt +++ b/ydb/core/kqp/provider/CMakeLists.darwin.txt @@ -18,6 +18,7 @@ target_link_libraries(core-kqp-provider PUBLIC yutil ydb-core-base ydb-core-protos + ydb-core-docapi ydb-library-aclib library-aclib-protos ydb-library-binary_json diff --git a/ydb/core/kqp/provider/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/provider/CMakeLists.linux-aarch64.txt index 53be60b082..f928da4305 100644 --- a/ydb/core/kqp/provider/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/provider/CMakeLists.linux-aarch64.txt @@ -19,6 +19,7 @@ target_link_libraries(core-kqp-provider PUBLIC yutil ydb-core-base ydb-core-protos + ydb-core-docapi ydb-library-aclib library-aclib-protos ydb-library-binary_json diff --git a/ydb/core/kqp/provider/CMakeLists.linux.txt b/ydb/core/kqp/provider/CMakeLists.linux.txt index 53be60b082..f928da4305 100644 --- a/ydb/core/kqp/provider/CMakeLists.linux.txt +++ b/ydb/core/kqp/provider/CMakeLists.linux.txt @@ -19,6 +19,7 @@ target_link_libraries(core-kqp-provider PUBLIC yutil ydb-core-base ydb-core-protos + ydb-core-docapi ydb-library-aclib library-aclib-protos ydb-library-binary_json diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 60e158cbcb..da91b8a9d5 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -1,5 +1,7 @@ #include "yql_kikimr_provider_impl.h" +#include <ydb/core/docapi/traits.h> + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/core/yql_execution.h> #include <ydb/library/yql/core/yql_graph_transformer.h> @@ -969,6 +971,8 @@ public: if (to_lower(format) == "json") { add_changefeed->set_format(Ydb::Table::ChangefeedFormat::FORMAT_JSON); + } else if (to_lower(format) == "dynamodb_streams_json") { + add_changefeed->set_format(Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), TStringBuilder() << "Unknown changefeed format: " << format)); @@ -1001,7 +1005,12 @@ public: const auto duration = TDuration::FromValue(value); auto& retention = *add_changefeed->mutable_retention_period(); retention.set_seconds(duration.Seconds()); - retention.set_nanos(duration.NanoSecondsOfSecond()); + } else if (name == "aws_region") { + auto value = TString( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + + add_changefeed->set_aws_region(value); } else if (name == "local") { // nop } else { @@ -1063,7 +1072,11 @@ public: } else if (isColumn) { future = Gateway->AlterColumnTable(cluster, ParseAlterColumnTableSettings(maybeAlter.Cast())); } else { - future = Gateway->AlterTable(std::move(alterTableRequest), cluster); + TMaybe<TString> requestType; + if (!SessionCtx->Query().DocumentApiRestricted) { + requestType = NKikimr::NDocApi::RequestType; + } + future = Gateway->AlterTable(cluster, std::move(alterTableRequest), requestType); } } @@ -1349,7 +1362,6 @@ private: bool ApplyTableOperations(const TString& cluster, const TVector<NKqpProto::TKqpTableOp>& tableOps, NKikimrKqp::EIsolationLevel isolationLevel, TExprContext& ctx) { - bool enableImmediateEffects = SessionCtx->Config().FeatureFlags.GetEnableKqpImmediateEffects(); auto queryType = SessionCtx->Query().Type; TVector<NKqpProto::TKqpTableInfo> tableInfo; @@ -1362,13 +1374,11 @@ private: } if (!SessionCtx->HasTx()) { - TKikimrTransactionContextBase emptyCtx; - return emptyCtx.ApplyTableOperations(tableOps, tableInfo, isolationLevel, enableImmediateEffects, - queryType, ctx); + TKikimrTransactionContextBase emptyCtx(SessionCtx->Config().EnableKqpImmediateEffects); + return emptyCtx.ApplyTableOperations(tableOps, tableInfo, isolationLevel, queryType, ctx); } - return SessionCtx->Tx().ApplyTableOperations(tableOps, tableInfo, isolationLevel, - enableImmediateEffects, queryType, ctx); + return SessionCtx->Tx().ApplyTableOperations(tableOps, tableInfo, isolationLevel, queryType, ctx); } bool ApplyDdlOperation(const TString& cluster, TPositionHandle pos, const TString& table, diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index c8bea55906..a0f5a0b3b4 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -620,7 +620,7 @@ public: virtual NThreading::TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir) = 0; - virtual NThreading::TFuture<TGenericResult> AlterTable(Ydb::Table::AlterTableRequest&& req, const TString& cluster) = 0; + virtual NThreading::TFuture<TGenericResult> AlterTable(const TString& cluster, Ydb::Table::AlterTableRequest&& req, const TMaybe<TString>& requestType) = 0; virtual NThreading::TFuture<TGenericResult> RenameTable(const TString& src, const TString& dst, const TString& cluster) = 0; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index 9079b5040b..6a34b5ee48 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -220,12 +220,9 @@ bool AddDmlIssue(const TIssue& issue, TExprContext& ctx); class TKikimrTransactionContextBase : public TThrRefBase { public: - THashMap<TString, TYdbOperations> TableOperations; - THashMap<TKikimrPathId, TString> TableByIdMap; - TMaybe<NKikimrKqp::EIsolationLevel> EffectiveIsolationLevel; - bool Readonly = false; - bool Invalidated = false; - bool Closed = false; + explicit TKikimrTransactionContextBase(bool enableImmediateEffects) + : EnableImmediateEffects(enableImmediateEffects) { + } bool HasStarted() const { return EffectiveIsolationLevel.Defined(); @@ -256,12 +253,13 @@ public: Invalidated = false; Readonly = false; Closed = false; + HasUncommittedChangesRead = false; } template<class IterableKqpTableOps, class IterableKqpTableInfos> bool ApplyTableOperations(const IterableKqpTableOps& operations, const IterableKqpTableInfos& tableInfos, NKikimrKqp::EIsolationLevel isolationLevel, - bool enableImmediateEffects, EKikimrQueryType queryType, TExprContext& ctx) + EKikimrQueryType queryType, TExprContext& ctx) { if (IsClosed()) { TString message = TStringBuilder() << "Cannot perform operations on closed transaction."; @@ -346,19 +344,28 @@ public: auto& currentOps = TableOperations[table]; bool currentModify = currentOps & KikimrModifyOps(); - if (currentModify && !enableImmediateEffects) { + if (currentModify) { if (KikimrReadOps() & newOp) { - TString message = TStringBuilder() << "Data modifications previously made to table '" << table - << "' in current transaction won't be seen by operation: '" << newOp << "'"; - if (!AddDmlIssue(YqlIssue(pos, TIssuesIds::KIKIMR_READ_MODIFIED_TABLE, message), ctx)) { - return false; + if (!EnableImmediateEffects) { + TString message = TStringBuilder() << "Data modifications previously made to table '" << table + << "' in current transaction won't be seen by operation: '" << newOp << "'"; + if (!AddDmlIssue(YqlIssue(pos, TIssuesIds::KIKIMR_READ_MODIFIED_TABLE, message), ctx)) { + return false; + } } + + HasUncommittedChangesRead = true; } if (info->GetHasIndexTables()) { - TString message = TStringBuilder() << "Multiple modification of table with secondary indexes is not supported yet"; - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_OPERATION, message)); - return false; + if (!EnableImmediateEffects) { + TString message = TStringBuilder() + << "Multiple modification of table with secondary indexes is not supported yet"; + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_OPERATION, message)); + return false; + } + + HasUncommittedChangesRead = true; } } @@ -370,6 +377,15 @@ public: virtual ~TKikimrTransactionContextBase() = default; +public: + THashMap<TString, TYdbOperations> TableOperations; + bool HasUncommittedChangesRead = false; + const bool EnableImmediateEffects; + THashMap<TKikimrPathId, TString> TableByIdMap; + TMaybe<NKikimrKqp::EIsolationLevel> EffectiveIsolationLevel; + bool Readonly = false; + bool Invalidated = false; + bool Closed = false; }; class TKikimrSessionContext : public TThrRefBase { diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 4168a1e664..69c9d8e74d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -136,6 +136,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnableKqpScanQueryStreamIdxLookupJoin = false; bool EnablePredicateExtractForScanQuery = true; bool EnablePredicateExtractForDataQuery = false; + bool EnableKqpImmediateEffects = false; }; } diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index a6706d8324..8b59caeea3 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -1,5 +1,7 @@ #include "yql_kikimr_provider_impl.h" +#include <ydb/core/docapi/traits.h> + #include <ydb/library/yql/core/type_ann/type_ann_impl.h> #include <ydb/library/yql/core/type_ann/type_ann_list.h> #include <ydb/library/yql/core/yql_expr_optimize.h> @@ -13,8 +15,6 @@ namespace { using namespace NCommon; using namespace NNodes; -const TString DocApiTableVersionAttribute = "__document_api_version"; - const TTypeAnnotationNode* GetExpectedRowType(const TKikimrTableDescription& tableDesc, const TVector<TString>& columns, const TPosition& pos, TExprContext& ctx) { @@ -1288,7 +1288,7 @@ private: return true; } - if (!meta.Attributes.FindPtr(DocApiTableVersionAttribute)) { + if (!meta.Attributes.FindPtr(NKikimr::NDocApi::VersionAttribute)) { return true; } diff --git a/ydb/core/kqp/runtime/kqp_read_actor.cpp b/ydb/core/kqp/runtime/kqp_read_actor.cpp index cc12f68b42..fa9feecb0f 100644 --- a/ydb/core/kqp/runtime/kqp_read_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_read_actor.cpp @@ -61,7 +61,8 @@ THolder<NKikimr::TEvDataShard::TEvReadAck> DefaultAckSettings() { return result; } -NActors::TActorId PipeCacheId = NKikimr::MakePipePeNodeCacheID(false); +NActors::TActorId MainPipeCacheId = NKikimr::MakePipePeNodeCacheID(false); +NActors::TActorId FollowersPipeCacheId = NKikimr::MakePipePeNodeCacheID(true); TDuration StartRetryDelay = TDuration::MilliSeconds(250); @@ -104,6 +105,7 @@ public: size_t ResolveAttempt = 0; size_t RetryAttempt = 0; + size_t SuccessBatches = 0; TShardState(ui64 tabletId) : TabletId(tabletId) @@ -360,6 +362,8 @@ public: , HolderFactory(args.HolderFactory) , Alloc(args.Alloc) , Counters(counters) + , UseFollowers(false) + , PipeCacheId(MainPipeCacheId) { TableId = TTableId( Settings.GetTable().GetTableId().GetOwnerId(), @@ -368,6 +372,13 @@ public: Settings.GetTable().GetTableId().GetSchemaVersion() ); + if (Settings.GetUseFollowers() && !Snapshot.IsValid()) { + // reading from followers is allowed only of snapshot is not specified and + // specific flag is set. otherwise we always read from main replicas. + PipeCacheId = FollowersPipeCacheId; + UseFollowers = true; + } + KeyColumnTypes.reserve(Settings.GetKeyColumnTypes().size()); for (size_t i = 0; i < Settings.KeyColumnTypesSize(); ++i) { auto typeId = Settings.GetKeyColumnTypes(i); @@ -823,7 +834,7 @@ public: Counters->CreatedIterators->Inc(); ReadIdByTabletId[state->TabletId].push_back(id); - Send(::PipeCacheId, new TEvPipeCache::TEvForward(ev.Release(), state->TabletId, true), + Send(PipeCacheId, new TEvPipeCache::TEvForward(ev.Release(), state->TabletId, true), IEventHandle::FlagTrackDelivery); } @@ -837,14 +848,9 @@ public: TString lastKey = "(empty)"; if (!token.GetLastProcessedKey().empty()) { TStringBuilder builder; - TVector<NScheme::TTypeInfo> types; - for (auto& column : Settings.GetColumns()) { - types.push_back(NScheme::TTypeInfo((NScheme::TTypeId)column.GetType())); - } - TSerializedCellVec row(token.GetLastProcessedKey()); - lastKey = DebugPrintPoint(types, row.GetCells(), *AppData()->TypeRegistry); + lastKey = DebugPrintPoint(KeyColumnTypes, row.GetCells(), *AppData()->TypeRegistry); } return TStringBuilder() << "first request = " << token.GetFirstUnprocessedQuery() << " lastkey = " << lastKey; } @@ -866,9 +872,20 @@ public: CA_LOG_D("read id #" << id << " got issue " << issue.Getmessage()); Reads[id].Shard->Issues.push_back(issue); } + + if (UseFollowers && record.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS && Reads[id].Shard->SuccessBatches > 0) { + // read from follower is interrupted with error after several successful responses. + // in this case read is not safe because we can return inconsistent data. + NYql::TIssues issues; + NYql::IssuesFromMessage(record.GetStatus().GetIssues(), issues); + return RuntimeError("Failed to read from follower", NYql::NDqProto::StatusIds::UNAVAILABLE, issues); + } + switch (record.GetStatus().GetCode()) { - case Ydb::StatusIds::SUCCESS: + case Ydb::StatusIds::SUCCESS: { + Reads[id].Shard->SuccessBatches++; break; + } case Ydb::StatusIds::OVERLOADED: { return RetryRead(id, false); } @@ -938,7 +955,7 @@ public: auto* state = Reads[id].Shard; auto cancel = MakeHolder<TEvDataShard::TEvReadCancel>(); cancel->Record.SetReadId(id); - Send(::PipeCacheId, new TEvPipeCache::TEvForward(cancel.Release(), state->TabletId, false)); + Send(PipeCacheId, new TEvPipeCache::TEvForward(cancel.Release(), state->TabletId, false)); Reads[id].Reset(); ResetReads++; @@ -1150,7 +1167,7 @@ public: } Counters->SentIteratorAcks->Inc(); CA_LOG_D("sending ack for read #" << id << " limit " << limit << " seqno = " << record.GetSeqNo()); - Send(::PipeCacheId, new TEvPipeCache::TEvForward(request.Release(), Reads[id].Shard->TabletId, true), + Send(PipeCacheId, new TEvPipeCache::TEvForward(request.Release(), Reads[id].Shard->TabletId, true), IEventHandle::FlagTrackDelivery); } else { Reads[id].Finished = true; @@ -1237,7 +1254,10 @@ public: for (size_t i = 0; i < Reads.size(); ++i) { ResetRead(i); } - Send(PipeCacheId, new TEvPipeCache::TEvUnlink(0)); + Send(::MainPipeCacheId, new TEvPipeCache::TEvUnlink(0)); + if (UseFollowers) { + Send(::FollowersPipeCacheId, new TEvPipeCache::TEvUnlink(0)); + } } TBase::PassAway(); } @@ -1317,6 +1337,8 @@ private: std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc> Alloc; TIntrusivePtr<TKqpCounters> Counters; + bool UseFollowers; + NActors::TActorId PipeCacheId; }; @@ -1338,7 +1360,7 @@ void InjectRangeEvReadAckSettings(const NKikimrTxDataShard::TEvReadAck& ack) { } void InterceptReadActorPipeCache(NActors::TActorId id) { - ::PipeCacheId = id; + ::MainPipeCacheId = id; } } // namespace NKqp diff --git a/ydb/core/kqp/session_actor/CMakeLists.darwin.txt b/ydb/core/kqp/session_actor/CMakeLists.darwin.txt index 44a02a9706..990161a080 100644 --- a/ydb/core/kqp/session_actor/CMakeLists.darwin.txt +++ b/ydb/core/kqp/session_actor/CMakeLists.darwin.txt @@ -14,6 +14,7 @@ target_compile_options(core-kqp-session_actor PRIVATE target_link_libraries(core-kqp-session_actor PUBLIC contrib-libs-cxxsupp yutil + ydb-core-docapi core-kqp-common ) target_sources(core-kqp-session_actor PRIVATE diff --git a/ydb/core/kqp/session_actor/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/session_actor/CMakeLists.linux-aarch64.txt index 1f590542cb..0990f751b6 100644 --- a/ydb/core/kqp/session_actor/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/session_actor/CMakeLists.linux-aarch64.txt @@ -15,6 +15,7 @@ target_link_libraries(core-kqp-session_actor PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + ydb-core-docapi core-kqp-common ) target_sources(core-kqp-session_actor PRIVATE diff --git a/ydb/core/kqp/session_actor/CMakeLists.linux.txt b/ydb/core/kqp/session_actor/CMakeLists.linux.txt index 1f590542cb..0990f751b6 100644 --- a/ydb/core/kqp/session_actor/CMakeLists.linux.txt +++ b/ydb/core/kqp/session_actor/CMakeLists.linux.txt @@ -15,6 +15,7 @@ target_link_libraries(core-kqp-session_actor PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + ydb-core-docapi core-kqp-common ) target_sources(core-kqp-session_actor PRIVATE diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 530a542f60..40b11c1388 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -419,7 +419,7 @@ public: QueryState->TxCtx = std::move(txCtx); QueryState->QueryData = std::make_shared<TQueryData>(QueryState->TxCtx->TxAlloc); QueryState->TxId = txId; - if (!CheckTransacionLocks()) { + if (!CheckTransactionLocks(/*tx*/ nullptr)) { return; } @@ -787,7 +787,8 @@ public: void BeginTx(const Ydb::Table::TransactionSettings& settings) { QueryState->TxId = UlidGen.Next(); - QueryState->TxCtx = MakeIntrusive<TKqpTransactionContext>(false, AppData()->FunctionRegistry, AppData()->TimeProvider, AppData()->RandomProvider); + QueryState->TxCtx = MakeIntrusive<TKqpTransactionContext>(false, AppData()->FunctionRegistry, + AppData()->TimeProvider, AppData()->RandomProvider, Config->EnableKqpImmediateEffects); QueryState->QueryData = std::make_shared<TQueryData>(QueryState->TxCtx->TxAlloc); QueryState->TxCtx->SetIsolationLevel(settings); QueryState->TxCtx->OnBeginQuery(); @@ -807,11 +808,10 @@ public: std::pair<bool, TIssues> ApplyTableOperations(TKqpTransactionContext* txCtx, const NKqpProto::TKqpPhyQuery& query) { auto isolationLevel = *txCtx->EffectiveIsolationLevel; - bool enableImmediateEffects = Config->FeatureFlags.GetEnableKqpImmediateEffects(); TExprContext ctx; bool success = txCtx->ApplyTableOperations(query.GetTableOps(), query.GetTableInfos(), isolationLevel, - enableImmediateEffects, EKikimrQueryType::Dml, ctx); + EKikimrQueryType::Dml, ctx); return {success, ctx.IssueManager.GetIssues()}; } @@ -844,7 +844,7 @@ public: } } else { QueryState->TxCtx = MakeIntrusive<TKqpTransactionContext>(false, AppData()->FunctionRegistry, - AppData()->TimeProvider, AppData()->RandomProvider); + AppData()->TimeProvider, AppData()->RandomProvider, Config->EnableKqpImmediateEffects); QueryState->QueryData = std::make_shared<TQueryData>(QueryState->TxCtx->TxAlloc); QueryState->TxCtx->EffectiveIsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED; } @@ -1077,11 +1077,15 @@ public: return false; } + if (txCtx.Locks.GetLockTxId() && !txCtx.Locks.Broken()) { + return true; // Continue to acquire locks + } + if (txCtx.Locks.Broken()) { return false; // Do not acquire locks after first lock issue } - if (!txCtx.DeferredEffects.Empty()) { + if (txCtx.TxHasEffects()) { return true; // Acquire locks in read write tx } @@ -1110,13 +1114,20 @@ public: return QueryState->QueryData; } - bool CheckTransacionLocks() { + bool CheckTransactionLocks(const TKqpPhyTxHolder::TConstPtr& tx) { auto& txCtx = *QueryState->TxCtx; if (!txCtx.DeferredEffects.Empty() && txCtx.Locks.Broken()) { ReplyQueryError(Ydb::StatusIds::ABORTED, "tx has deferred effects, but locks are broken", MessageFromIssues(std::vector<TIssue>{txCtx.Locks.GetIssue()})); return false; } + + if (tx && tx->GetHasEffects() && txCtx.Locks.Broken()) { + ReplyQueryError(Ydb::StatusIds::ABORTED, "tx has effects, but locks are broken", + MessageFromIssues(std::vector<TIssue>{txCtx.Locks.GetIssue()})); + return false; + } + return true; } @@ -1136,22 +1147,11 @@ public: return false; } - void ExecuteOrDefer() { + TKqpPhyTxHolder::TConstPtr GetCurrentPhyTx(const NKqpProto::TKqpPhyQuery& phyQuery) { auto& txCtx = *QueryState->TxCtx; - - bool haveWork = QueryState->PreparedQuery && - QueryState->CurrentTx < QueryState->PreparedQuery->GetPhysicalQuery().TransactionsSize() - || QueryState->Commit && !QueryState->Commited; - - if (!haveWork) { - ReplySuccess(); - return; - } - - const auto& phyQuery = QueryState->PreparedQuery->GetPhysicalQuery(); - auto tx = QueryState->PreparedQuery->GetPhyTxOrEmpty(QueryState->CurrentTx); - if (!Config->FeatureFlags.GetEnableKqpImmediateEffects()) { + + if (txCtx.CanDeferEffects()) { while (tx && tx->GetHasEffects()) { YQL_ENSURE(txCtx.AddDeferredEffect(tx, CreateKqpValueMap(tx))); LWTRACK(KqpSessionPhyQueryDefer, QueryState->Orbit, QueryState->CurrentTx); @@ -1160,39 +1160,96 @@ public: tx = QueryState->PreparedQuery->GetPhyTx(QueryState->CurrentTx); } else { tx = nullptr; - break; } } } - if (!CheckTransacionLocks() || !CheckTopicOperations()) { + return tx; + } + + void ExecuteOrDefer() { + bool haveWork = QueryState->PreparedQuery && + QueryState->CurrentTx < QueryState->PreparedQuery->GetPhysicalQuery().TransactionsSize() + || QueryState->Commit && !QueryState->Commited; + + if (!haveWork) { + ReplySuccess(); return; } - bool commit = false; - if (QueryState->Commit && Config->FeatureFlags.GetEnableKqpImmediateEffects() && phyQuery.GetHasUncommittedChangesRead()) { - // every phy tx should acquire LockTxId, so commit is sent separately at the end - commit = QueryState->CurrentTx >= phyQuery.TransactionsSize(); - } else if (QueryState->Commit && QueryState->CurrentTx >= phyQuery.TransactionsSize() - 1) { - if (!tx) { - // no physical transactions left, perform commit - commit = true; - } else { - // we can merge commit with last tx only for read-only transactions - commit = txCtx.DeferredEffects.Empty(); - } + const auto& phyQuery = QueryState->PreparedQuery->GetPhysicalQuery(); + auto tx = GetCurrentPhyTx(phyQuery); + + if (!CheckTransactionLocks(tx) || !CheckTopicOperations()) { + return; } - if (tx || commit) { - bool replied = ExecutePhyTx(&phyQuery, tx, commit); - if (!replied) { - ++QueryState->CurrentTx; - } + if (QueryState->TxCtx->ShouldExecuteDeferredEffects()) { + ExecuteDeferredEffectsImmediately(); + } else if (auto commit = ShouldCommitWithCurrentTx(phyQuery, tx); commit || tx) { + ExecutePhyTx(&phyQuery, tx, commit); } else { ReplySuccess(); } } + void ExecuteDeferredEffectsImmediately() { + YQL_ENSURE(QueryState->TxCtx->ShouldExecuteDeferredEffects()); + + auto& txCtx = *QueryState->TxCtx; + auto request = PrepareRequest(/* tx */ nullptr, /* literal */ false, QueryState.get()); + + for (const auto& effect : txCtx.DeferredEffects) { + YQL_ENSURE(effect.PhysicalTx->GetType() == NKqpProto::TKqpPhyTx::TYPE_DATA); + request.Transactions.emplace_back(effect.PhysicalTx, effect.Params); + + LOG_D("TExecPhysicalRequest, add DeferredEffect to Transaction," + << " current Transactions.size(): " << request.Transactions.size()); + } + + request.AcquireLocksTxId = txCtx.Locks.GetLockTxId(); + request.UseImmediateEffects = true; + request.PerShardKeysSizeLimitBytes = Config->_CommitPerShardKeysSizeLimitBytes.Get().GetRef(); + + txCtx.HasImmediateEffects = true; + txCtx.ClearDeferredEffects(); + + SendToExecuter(std::move(request)); + } + + + bool ShouldCommitWithCurrentTx(const NKqpProto::TKqpPhyQuery& phyQuery, const TKqpPhyTxHolder::TConstPtr& tx) { + if (!QueryState->Commit) { + return false; + } + + if (QueryState->CurrentTx + 1 < phyQuery.TransactionsSize()) { + // commit can only be applied to the last transaction or perform separately at the end + return false; + } + + if (!tx) { + // no physical transactions left, perform commit + return true; + } + + if (QueryState->TxCtx->HasUncommittedChangesRead) { + YQL_ENSURE(QueryState->TxCtx->EnableImmediateEffects); + + if (tx && tx->GetHasEffects()) { + YQL_ENSURE(tx->ResultsSize() == 0); + // commit can be applied to the last transaction with effects + return QueryState->CurrentTx + 1 == phyQuery.TransactionsSize(); + } + + // last tx contains reads, so commit should be sent separately + return false; + } + + // we can merge commit with last tx only for read-only transactions + return !QueryState->TxCtx->TxHasEffects(); + } + bool ExecutePhyTx(const NKqpProto::TKqpPhyQuery* query, const TKqpPhyTxHolder::TConstPtr& tx, bool commit) { auto& txCtx = *QueryState->TxCtx; @@ -1272,6 +1329,10 @@ public: request.TopicOperations = std::move(txCtx.TopicOperations); } else if (ShouldAcquireLocks(query)) { request.AcquireLocksTxId = txCtx.Locks.GetLockTxId(); + if (txCtx.HasUncommittedChangesRead) { + YQL_ENSURE(txCtx.EnableImmediateEffects); + request.UseImmediateEffects = true; + } } LWTRACK(KqpSessionPhyQueryProposeTx, @@ -1282,7 +1343,7 @@ public: request.AcquireLocksTxId.Defined()); SendToExecuter(std::move(request)); - + ++QueryState->CurrentTx; return false; } @@ -1421,7 +1482,10 @@ public: auto& executerResults = *response->MutableResult(); { auto g = QueryState->QueryData->TypeEnv().BindAllocator(); - QueryState->QueryData->AddTxResults(std::move(ev->GetTxResults())); + if (!ev->GetTxResults().empty()) { + QueryState->QueryData->AddTxResults(QueryState->CurrentTx - 1, std::move(ev->GetTxResults())); + } + QueryState->QueryData->AddTxHolders(std::move(ev->GetTxHolders())); } diff --git a/ydb/core/kqp/session_actor/kqp_tx.cpp b/ydb/core/kqp/session_actor/kqp_tx.cpp index 201e051cd8..7dc3b06a09 100644 --- a/ydb/core/kqp/session_actor/kqp_tx.cpp +++ b/ydb/core/kqp/session_actor/kqp_tx.cpp @@ -41,13 +41,14 @@ std::pair<bool, std::vector<TIssue>> MergeLocks(const NKikimrMiniKQL::TType& typ YQL_ENSURE(locksListType.GetItem().GetKind() == NKikimrMiniKQL::ETypeKind::Struct); auto lockType = locksListType.GetItem().GetStruct(); - YQL_ENSURE(lockType.MemberSize() == 6); + YQL_ENSURE(lockType.MemberSize() == 7); YQL_ENSURE(lockType.GetMember(0).GetName() == "Counter"); YQL_ENSURE(lockType.GetMember(1).GetName() == "DataShard"); YQL_ENSURE(lockType.GetMember(2).GetName() == "Generation"); YQL_ENSURE(lockType.GetMember(3).GetName() == "LockId"); YQL_ENSURE(lockType.GetMember(4).GetName() == "PathId"); YQL_ENSURE(lockType.GetMember(5).GetName() == "SchemeShard"); + YQL_ENSURE(lockType.GetMember(6).GetName() == "HasWrites"); res.first = true; for (auto& lockValue : value.GetList()) { @@ -65,6 +66,10 @@ std::pair<bool, std::vector<TIssue>> MergeLocks(const NKikimrMiniKQL::TType& typ res.first = false; } else if (auto curTxLock = locks.LocksMap.FindPtr(txLock.GetKey())) { + if (txLock.HasWrites()) { + curTxLock->SetHasWrites(); + } + if (curTxLock->Invalidated(txLock)) { res.second.emplace_back(GetLocksInvalidatedIssue(txCtx, txLock)); res.first = false; @@ -171,7 +176,8 @@ bool NeedSnapshot(const TKqpTransactionContext& txCtx, const NYql::TKikimrConfig } } - if (config.FeatureFlags.GetEnableKqpImmediateEffects() && physicalQuery.GetHasUncommittedChangesRead()) { + if (txCtx.HasUncommittedChangesRead) { + YQL_ENSURE(txCtx.EnableImmediateEffects); return true; } diff --git a/ydb/core/kqp/session_actor/kqp_tx.h b/ydb/core/kqp/session_actor/kqp_tx.h index 0d63338bb7..0a2de1a805 100644 --- a/ydb/core/kqp/session_actor/kqp_tx.h +++ b/ydb/core/kqp/session_actor/kqp_tx.h @@ -25,6 +25,10 @@ public: ui64 GetPathId() const { return LockValue.GetStruct(4).GetUint64(); } ui32 GetGeneration() const { return LockValue.GetStruct(2).GetUint32(); } ui64 GetCounter() const { return LockValue.GetStruct(0).GetUint64(); } + bool HasWrites() const { return LockValue.GetStruct(6).GetBool(); } + void SetHasWrites() { + LockValue.MutableStruct(6)->SetBool(true); + } TKey GetKey() const { return std::make_tuple(GetLockId(), GetDataShard(), GetSchemeShard(), GetPathId()); } NKikimrMiniKQL::TValue GetValue() const { return LockValue; } @@ -119,8 +123,9 @@ private: class TKqpTransactionContext : public NYql::TKikimrTransactionContextBase { public: explicit TKqpTransactionContext(bool implicit, const NMiniKQL::IFunctionRegistry* funcRegistry, - TIntrusivePtr<ITimeProvider> timeProvider, TIntrusivePtr<IRandomProvider> randomProvider) - : Implicit(implicit) + TIntrusivePtr<ITimeProvider> timeProvider, TIntrusivePtr<IRandomProvider> randomProvider, bool enableImmediateEffects) + : NYql::TKikimrTransactionContextBase(enableImmediateEffects) + , Implicit(implicit) , ParamsState(MakeIntrusive<TParamsState>()) { CreationTime = TInstant::Now(); @@ -217,6 +222,24 @@ public: }; } + bool ShouldExecuteDeferredEffects() const { + if (HasUncommittedChangesRead) { + YQL_ENSURE(EnableImmediateEffects); + return !DeferredEffects.Empty(); + } + + return false; + } + + bool CanDeferEffects() const { + if (HasUncommittedChangesRead) { + YQL_ENSURE(EnableImmediateEffects); + return false; + } + + return true; + } + public: struct TParamsState : public TThrRefBase { ui32 LastIndex = 0; diff --git a/ydb/core/kqp/session_actor/kqp_worker_actor.cpp b/ydb/core/kqp/session_actor/kqp_worker_actor.cpp index 0e63e5ad0a..75f84fa3d5 100644 --- a/ydb/core/kqp/session_actor/kqp_worker_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_worker_actor.cpp @@ -595,7 +595,9 @@ private: switch (type) { case NKikimrKqp::QUERY_TYPE_SQL_DDL: { - QueryState->AsyncQueryResult = KqpHost->ExecuteSchemeQuery(query, true); + IKqpHost::TExecSettings execSettings; + execSettings.DocumentApiRestricted = IsDocumentApiRestricted(QueryState->RequestType); + QueryState->AsyncQueryResult = KqpHost->ExecuteSchemeQuery(query, true, execSettings); break; } diff --git a/ydb/core/kqp/session_actor/kqp_worker_common.h b/ydb/core/kqp/session_actor/kqp_worker_common.h index 507256a315..ada75468c1 100644 --- a/ydb/core/kqp/session_actor/kqp_worker_common.h +++ b/ydb/core/kqp/session_actor/kqp_worker_common.h @@ -1,5 +1,6 @@ #include "kqp_session_actor.h" +#include <ydb/core/docapi/traits.h> #include <ydb/core/kqp/common/kqp.h> #include <ydb/core/kqp/provider/yql_kikimr_gateway.h> #include <ydb/core/kqp/provider/yql_kikimr_provider.h> @@ -129,7 +130,7 @@ void SlowLogQuery(const TActorContext &ctx, const NYql::TKikimrConfiguration* co NYql::TKikimrQueryLimits GetQueryLimits(const TKqpWorkerSettings& settings); inline bool IsDocumentApiRestricted(const TString& requestType) { - return requestType != "_document_api_request"sv; + return requestType != NDocApi::RequestType; } TMaybe<Ydb::StatusIds::StatusCode> GetYdbStatus(const NYql::TIssue& issue); diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 9a4887b3c9..7409c059c1 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -102,12 +102,9 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { ServerSettings->SetEnableNotNullColumns(true); ServerSettings->SetEnableMoveIndex(true); - if (settings.FeatureFlags.GetEnableKqpImmediateEffects()) { + if (settings.AppConfig.GetTableServiceConfig().GetEnableKqpImmediateEffects()) { Tests::TServerSettings::TControls controls; - controls.MutableDataShardControls()->SetPrioritizedMvccSnapshotReads(1); - controls.MutableDataShardControls()->SetUnprotectedMvccSnapshotReads(1); controls.MutableDataShardControls()->SetEnableLockedWrites(1); - ServerSettings->SetControls(controls); } diff --git a/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp b/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp index 86c40267f4..bf66ee01bd 100644 --- a/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp +++ b/ydb/core/kqp/ut/effects/kqp_immediate_effects_ut.cpp @@ -1,4 +1,5 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> +#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> namespace NKikimr { namespace NKqp { @@ -27,12 +28,38 @@ namespace { )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } + + void CreateShardedTestTable(TSession& session) { + AssertSuccessResult(session.ExecuteSchemeQuery(R"( + --!syntax_v1 + + CREATE TABLE TestImmediateEffects ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) WITH ( + PARTITION_AT_KEYS = (100, 200) + ); + )").GetValueSync()); + + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + INSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "Value1"), + (2u, "Value2"), + (100u, "Value100"), + (200u, "Value200"); + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } } // namespase Y_UNIT_TEST_SUITE(KqpImmediateEffects) { Y_UNIT_TEST(Upsert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -87,8 +114,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpsertDuplicates) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -120,8 +148,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpsertExistingKey) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -166,8 +195,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(Replace) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -222,8 +252,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(ReplaceDuplicates) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -255,8 +286,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(ReplaceExistingKey) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -301,8 +333,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(Insert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -357,8 +390,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(InsertDuplicates) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -384,8 +418,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(InsertExistingKey) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -410,8 +445,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpdateOn) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -462,8 +498,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(Delete) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -510,8 +547,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpdateAfterUpsert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -539,8 +577,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(DeleteAfterUpsert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -577,8 +616,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpdateAfterInsert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -606,8 +646,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(DeleteAfterInsert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -645,8 +686,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpsertAfterInsert) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -670,8 +712,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(UpsertAfterInsertWithIndex) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -745,8 +788,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(DeleteOnAfterInsertWithIndex) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -806,8 +850,9 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { } Y_UNIT_TEST(MultipleEffectsWithIndex) { - auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(true); + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -844,6 +889,1232 @@ Y_UNIT_TEST_SUITE(KqpImmediateEffects) { CompareYson(FormatResultSetYson(result.GetResultSet(0)), FormatResultSetYson(result.GetResultSet(3))); CompareYson(FormatResultSetYson(result.GetResultSet(1)), FormatResultSetYson(result.GetResultSet(2))); } + + Y_UNIT_TEST(InsertConflictTxAborted) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"); + + INSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "NewValue3"), + (201u, "Value201"); + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(UpsertConflictInteractiveTxAborted) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx; + { + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx = result.GetTransaction(); + UNIT_ASSERT(tx); + } + + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "NewValue3"); + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = tx->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[3u];["Value3"]]; + [[100u];["Value100"]]; + [[101u];["Value101"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(MultiShardUpsertAfterRead) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + TMaybe<TTransaction> tx; + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx = result.GetTransaction(); + UNIT_ASSERT(tx); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"); + )", TTxControl::Tx(*tx).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(TxWithReadAtTheEnd) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"), + (201u, "Value201"); + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (4u, "Value4"), + (101u, "NewValue101"); + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[3u];["Value3"]]; + [[4u];["Value4"]]; + [[100u];["Value100"]]; + [[101u];["NewValue101"]]; + [[200u];["Value200"]]; + [[201u];["Value201"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + // check that last (commit) phase is empty + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(stats.query_phases().size() - 1).table_access().size(), 0); + } + + Y_UNIT_TEST(InteractiveTxWithReadAtTheEnd) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); + + TMaybe<TTransaction> tx; + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"), + (201u, "Value201"); + )", TTxControl::BeginTx(), execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx = result.GetTransaction(); + UNIT_ASSERT(tx); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (4u, "Value4"); + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::Tx(*tx).CommitTx(), execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[3u];["Value3"]]; + [[4u];["Value4"]]; + [[100u];["Value100"]]; + [[101u];["Value101"]]; + [[200u];["Value200"]]; + [[201u];["Value201"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + // check that last (commit) phase is empty + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(stats.query_phases().size() - 1).table_access().size(), 0); + } + } + + Y_UNIT_TEST(TxWithWriteAtTheEnd) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"), + (201u, "Value201"); + + SELECT * FROM TestImmediateEffects ORDER BY Key; + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (4u, "Value4"), + (101u, "NewValue101"); + )", TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[3u];["Value3"]]; + [[100u];["Value100"]]; + [[101u];["Value101"]]; + [[200u];["Value200"]]; + [[201u];["Value201"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + // check that last (commit) phase contains write operation + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(stats.query_phases().size() - 1).table_access().size(), 1); + } + + Y_UNIT_TEST(InteractiveTxWithWriteAtTheEnd) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); + + TMaybe<TTransaction> tx; + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (3u, "Value3"), + (101u, "Value101"), + (201u, "Value201"); + )", TTxControl::BeginTx(), execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx = result.GetTransaction(); + UNIT_ASSERT(tx); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (4u, "Value4"); + )", TTxControl::Tx(*tx).CommitTx(), execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[3u];["Value3"]]; + [[100u];["Value100"]]; + [[101u];["Value101"]]; + [[200u];["Value200"]]; + [[201u];["Value201"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + + auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + // check that last (commit) phase contains write operation + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(stats.query_phases().size() - 1).table_access().size(), 1); + } + } + + Y_UNIT_TEST(UnobservedUncommittedChangeConflict) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + TMaybe<TTransaction> tx1; + auto session1 = db.CreateSession().GetValueSync().GetSession(); + { + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (2u, "Value2Modified"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + TMaybe<TTransaction> tx2; + auto session2 = db.CreateSession().GetValueSync().GetSession(); + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (2u, "Value2MoreModified"); + )", TTxControl::Tx(*tx2)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = tx2->Commit().ExtractValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]]; + [[2u];["Value2Modified"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(AlreadyBrokenImmediateEffects) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + TMaybe<TTransaction> tx1; + auto session1 = db.CreateSession().GetValueSync().GetSession(); + { + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 100u; + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "Value1Modified"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + TMaybe<TTransaction> tx2; + auto session2 = db.CreateSession().GetValueSync().GetSession(); + { + // This just establishes a snapshot that is before tx1 commit + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 100u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * From TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + + { + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "Value1Impossible"); + )", TTxControl::Tx(*tx2)).ExtractValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + + { + auto result = tx2->Commit().ExtractValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects ORDER BY Key; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1Modified"]]; + [[2u];["Value2"]]; + [[100u];["Value100"]]; + [[200u];["Value200"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(WriteThenReadWithCommit) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session); + + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + + auto session1 = db.CreateSession().GetValueSync().GetSession(); + TMaybe<TTransaction> tx1; + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "ModifiedValue1"); + )", TTxControl::Tx(*tx1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx1).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["ModifiedValue1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyR1WR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // write2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyR1RWR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // read2 + write2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyR1WRR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // write2 + read2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyW1RR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // read2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyW1WR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // write2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue11"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue11"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyW1RWR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // read2 + write2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(ConflictingKeyW1WRR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // write2 + read2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(ConflictingKeyRW1RR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // read2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(ConflictingKeyRW1WR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // write2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue11"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(ConflictingKeyRW1RWR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // read2 + write2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue11"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(ConflictingKeyRW1WRR2) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(true); + auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session1 = db.CreateSession().GetValueSync().GetSession(); + auto session2 = db.CreateSession().GetValueSync().GetSession(); + + CreateShardedTestTable(session1); + + TMaybe<TTransaction> tx1; + TMaybe<TTransaction> tx2; + + { // read1 + write1 + auto result = session1.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue1"); + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["Value1"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx1 = result.GetTransaction(); + UNIT_ASSERT(tx1); + } + + { // write2 + read2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + UPSERT INTO TestImmediateEffects (Key, Value) VALUES + (1u, "NewValue11"); + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::BeginTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1u];["NewValue11"]] + ])", FormatResultSetYson(result.GetResultSet(0))); + tx2 = result.GetTransaction(); + UNIT_ASSERT(tx2); + } + + { // commit1 + auto result = tx1->Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // read2 + commit2 + auto result = session2.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT * FROM TestImmediateEffects WHERE Key = 1u; + )", TTxControl::Tx(*tx2).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString()); + } + } + } } // namespace NKqp diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index afae447c59..3940519fb8 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -1374,6 +1374,55 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } #endif + Y_UNIT_TEST(PredicatePushdown_LikeNotPushedDownForStringType) { + auto settings = TKikimrSettings() + .SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + + TStreamExecScanQuerySettings scanSettings; + scanSettings.Explain(true); + + TTableWithNullsHelper(kikimr).CreateTableWithNulls(); + WriteTestDataForTableWithNulls(kikimr, "/Root/tableWithNulls"); + EnableDebugLogging(kikimr); + + auto tableClient = kikimr.GetTableClient(); + auto query = R"(SELECT id, binary_str FROM `/Root/tableWithNulls` WHERE binary_str LIKE "5%")"; + auto it = tableClient.StreamExecuteScanQuery(query, scanSettings).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + auto result = CollectStreamResult(it); + auto ast = result.QueryStats->Getquery_ast(); + UNIT_ASSERT_C(ast.find("KqpOlapFilter") == std::string::npos, + TStringBuilder() << "Predicate pushed down. Query: " << query); + } + + Y_UNIT_TEST(PredicatePushdown_LikeNotPushedDownIfAnsiLikeDisabled) { + auto settings = TKikimrSettings() + .SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + + TStreamExecScanQuerySettings scanSettings; + scanSettings.Explain(true); + + TTableWithNullsHelper(kikimr).CreateTableWithNulls(); + WriteTestDataForTableWithNulls(kikimr, "/Root/tableWithNulls"); + EnableDebugLogging(kikimr); + + auto tableClient = kikimr.GetTableClient(); + auto query = R"( + PRAGMA DisableAnsiLike; + SELECT id, resource_id FROM `/Root/tableWithNulls` WHERE resource_id LIKE "%5%" + )"; + auto it = tableClient.StreamExecuteScanQuery(query, scanSettings).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + auto result = CollectStreamResult(it); + auto ast = result.QueryStats->Getquery_ast(); + UNIT_ASSERT_C(ast.find("KqpOlapFilter") == std::string::npos, + TStringBuilder() << "Predicate pushed down. Query: " << query); + } + Y_UNIT_TEST(PredicatePushdown_MixStrictAndNotStrict) { auto settings = TKikimrSettings() .SetWithSampleTables(false); diff --git a/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp b/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp index e2a95cdafe..460eed63c6 100644 --- a/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_not_null_ut.cpp @@ -1009,6 +1009,176 @@ Y_UNIT_TEST_SUITE(KqpNotNullColumns) { UNIT_ASSERT_VALUES_EQUAL_C(mainTableNotNullColumns, indexTableNotNullColumns, "Not null columns mismatch"); } } + + Y_UNIT_TEST(OptionalParametersDataQuery) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"( + CREATE TABLE `/Root/TestTable` ( + Key1 Utf8 NOT NULL, + Key2 Int32 NOT NULL, + Value1 Utf8, + Value2 Utf8, + PRIMARY KEY (Key1, Key2)); + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + UPSERT INTO `/Root/TestTable` (Key1, Key2, Value1, Value2) VALUES + ('One', 1, NULL, NULL), + ('Two', 2, 'Value2', 'Value20'), + ('Three', 3, NULL, 'Value30'); + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // select by full pk + const auto query = Q1_(R"( + DECLARE $key1 AS Utf8?; + DECLARE $key2 AS Int32?; + + SELECT a.Key1, a.Key2, a.Value1, a.Value2 + FROM TestTable AS a + WHERE a.Key1 = $key1 AND a.Key2 = $key2; + )"); + + auto params = TParamsBuilder() + .AddParam("$key1").OptionalUtf8("One").Build() + .AddParam("$key2").OptionalInt32(1).Build() + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + CompareYson(R"([["One";1;#;#]])", FormatResultSetYson(result.GetResultSet(0))); + } + + { // select by key1 + const auto query = Q1_(R"( + DECLARE $key1 AS Utf8?; + + SELECT a.Key1, a.Key2, a.Value1, a.Value2 + FROM TestTable AS a + WHERE a.Key1 = $key1; + )"); + + auto params = TParamsBuilder() + .AddParam("$key1").OptionalUtf8("Two").Build() + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + CompareYson(R"([["Two";2;["Value2"];["Value20"]]])", FormatResultSetYson(result.GetResultSet(0))); + } + + { // select by key2 + const auto query = Q1_(R"( + DECLARE $key2 AS Int32?; + + SELECT a.Key1, a.Key2, a.Value1, a.Value2 + FROM TestTable AS a + WHERE a.Key2 = $key2; + )"); + + auto params = TParamsBuilder() + .AddParam("$key2").OptionalInt32(3).Build() + .Build(); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + CompareYson(R"([["Three";3;#;["Value30"]]])", FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(OptionalParametersScanQuery) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + auto result = session.ExecuteSchemeQuery(R"( + CREATE TABLE `/Root/TestTable` ( + Key1 Utf8 NOT NULL, + Key2 Int32 NOT NULL, + Value1 Utf8, + Value2 Utf8, + PRIMARY KEY (Key1, Key2)); + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + UPSERT INTO `/Root/TestTable` (Key1, Key2, Value1, Value2) VALUES + ('One', 1, NULL, NULL), + ('Two', 2, 'Value2', 'Value20'), + ('Three', 3, NULL, 'Value30'); + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { // select by full pk + const auto query = Q1_(R"( + DECLARE $key1 AS Utf8?; + DECLARE $key2 AS Int32?; + + SELECT a.Key1, a.Key2, a.Value1, a.Value2 + FROM TestTable AS a + WHERE a.Key1 = $key1 AND a.Key2 = $key2; + )"); + + auto params = TParamsBuilder() + .AddParam("$key1").OptionalUtf8("One").Build() + .AddParam("$key2").OptionalInt32(1).Build() + .Build(); + + auto it = client.StreamExecuteScanQuery(query, std::move(params)).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([["One";1;#;#]])", StreamResultToYson(it)); + } + + { // select by key1 + const auto query = Q1_(R"( + DECLARE $key1 AS Utf8?; + + SELECT a.Key1, a.Key2, a.Value1, a.Value2 + FROM TestTable AS a + WHERE a.Key1 = $key1; + )"); + + auto params = TParamsBuilder() + .AddParam("$key1").OptionalUtf8("Two").Build() + .Build(); + + auto it = client.StreamExecuteScanQuery(query, std::move(params)).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([["Two";2;["Value2"];["Value20"]]])", StreamResultToYson(it)); + } + + { // select by key2 + const auto query = Q1_(R"( + DECLARE $key2 AS Int32?; + + SELECT a.Key1, a.Key2, a.Value1, a.Value2 + FROM TestTable AS a + WHERE a.Key2 = $key2; + )"); + + auto params = TParamsBuilder() + .AddParam("$key2").OptionalInt32(3).Build() + .Build(); + + auto it = client.StreamExecuteScanQuery(query, std::move(params)).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([["Three";3;#;["Value30"]]])", StreamResultToYson(it)); + } + } } } // namespace NKqp diff --git a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp index 4383c55383..6d62e7e450 100644 --- a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp @@ -83,6 +83,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { UNIT_ASSERT(ValidatePlanNodeIds(plan)); auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter"); + if (!join.IsDefined()) { + join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"); + } UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); @@ -112,6 +115,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { UNIT_ASSERT(ValidatePlanNodeIds(plan)); auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter"); + if (!join.IsDefined()) { + join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"); + } UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); @@ -194,6 +200,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) { auto res = CollectStreamResult(it); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); + Cerr << *res.PlanJson; Cerr << *res.PlanJson << Endl; @@ -206,6 +213,13 @@ Y_UNIT_TEST_SUITE(KqpExplain) { "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter" ); + if (!join.IsDefined()) { + join = FindPlanNodeByKv( + plan, + "Node Type", + "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan" + ); + } UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); diff --git a/ydb/core/kqp/ut/query/kqp_query_ut.cpp b/ydb/core/kqp/ut/query/kqp_query_ut.cpp index 636e93a7a1..ee510c9526 100644 --- a/ydb/core/kqp/ut/query/kqp_query_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_query_ut.cpp @@ -309,8 +309,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) { Y_UNIT_TEST_TWIN(QueryClientTimeout, EnableImmediateEffects) { NKikimrConfig::TAppConfig app; app.MutableTableServiceConfig()->SetEnableKqpDataQuerySourceRead(false); + app.MutableTableServiceConfig()->SetEnableKqpImmediateEffects(EnableImmediateEffects); auto serverSettings = TKikimrSettings() - .SetEnableKqpImmediateEffects(EnableImmediateEffects) .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 746433ec81..5697cd6470 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -2653,6 +2653,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) { switch (format) { case EChangefeedFormat::Json: return "JSON"; + case EChangefeedFormat::DynamoDBStreamsJson: + return "DYNAMODB_STREAMS_JSON"; case EChangefeedFormat::Unknown: UNIT_ASSERT(false); return ""; @@ -2660,24 +2662,31 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void AddChangefeed(EChangefeedMode mode, EChangefeedFormat format) { - TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig())); + TKikimrRunner kikimr(TKikimrSettings() + .SetPQConfig(DefaultPQConfig()) + .SetEnableChangefeedDynamoDBStreamsFormat(true)); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); { - auto query = R"( - --!syntax_v1 - CREATE TABLE `/Root/table` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - )"; + auto builder = TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .SetPrimaryKeyColumn("Key"); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + if (format == EChangefeedFormat::DynamoDBStreamsJson) { + builder.AddAttribute("__document_api_version", "1"); + } + + auto result = session.CreateTable("/Root/table", builder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } + auto execOpts = TExecSchemeQuerySettings(); + if (format == EChangefeedFormat::DynamoDBStreamsJson) { + execOpts.RequestType("_document_api_request"); + } + { auto query = Sprintf(R"( --!syntax_v1 @@ -2686,7 +2695,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { ); )", ModeToString(mode), FormatToString(format)); - const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + const auto result = session.ExecuteSchemeQuery(query, execOpts).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto describeResult = session.DescribeTable("/Root/table").GetValueSync(); @@ -2704,7 +2713,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { ALTER TABLE `/Root/table` DROP CHANGEFEED `feed`; )"; - const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + const auto result = session.ExecuteSchemeQuery(query, execOpts).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } } @@ -2716,8 +2725,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } for (auto format : GetEnumAllValues<EChangefeedFormat>()) { - if (format == EChangefeedFormat::Unknown) { + switch (format) { + case EChangefeedFormat::Unknown: continue; + case EChangefeedFormat::DynamoDBStreamsJson: + if (mode == EChangefeedMode::Updates) { + continue; + } + break; + default: + break; } AddChangefeed(mode, format); @@ -2760,7 +2777,9 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } Y_UNIT_TEST(AddChangefeedNegative) { - TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig())); + TKikimrRunner kikimr(TKikimrSettings() + .SetPQConfig(DefaultPQConfig()) + .SetEnableChangefeedDynamoDBStreamsFormat(true)); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -2825,6 +2844,67 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); } + + { + auto result = session.CreateTable("/Root/document-table", TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .SetPrimaryKeyColumn("Key") + .AddAttribute("__document_api_version", "1") + .Build() + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto query = R"( + --!syntax_v1 + ALTER TABLE `/Root/document-table` ADD CHANGEFEED `feed` WITH ( + MODE = 'UPDATES', FORMAT = 'DYNAMODB_STREAMS_JSON' + ); + )"; + + const auto result = session.ExecuteSchemeQuery(query, TExecSchemeQuerySettings().RequestType("_document_api_request")).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(ChangefeedAwsRegion) { + TKikimrRunner kikimr(TKikimrSettings() + .SetPQConfig(DefaultPQConfig()) + .SetEnableChangefeedDynamoDBStreamsFormat(true)); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto result = session.CreateTable("/Root/table", TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .SetPrimaryKeyColumn("Key") + .AddAttribute("__document_api_version", "1") + .Build() + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto query = R"( + --!syntax_v1 + ALTER TABLE `/Root/table` ADD CHANGEFEED `feed` WITH ( + MODE = 'NEW_AND_OLD_IMAGES', FORMAT = 'DYNAMODB_STREAMS_JSON', AWS_REGION = 'aws:region' + ); + )"; + + const auto result = session.ExecuteSchemeQuery(query, TExecSchemeQuerySettings().RequestType("_document_api_request")).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto describeResult = session.DescribeTable("/Root/table").GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); + + const auto& changefeeds = describeResult.GetTableDescription().GetChangefeedDescriptions(); + UNIT_ASSERT_VALUES_EQUAL(changefeeds.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(changefeeds.at(0).GetAwsRegion(), "aws:region"); + } } Y_UNIT_TEST(DropChangefeedNegative) { @@ -2946,6 +3026,42 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } + Y_UNIT_TEST(ChangefeedAttributes) { + TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig())); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().ExtractValueSync().GetSession(); + + { + auto result = session.CreateTable("/Root/table", TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint64) + .AddNullableColumn("Value", EPrimitiveType::String) + .SetPrimaryKeyColumn("Key") + .Build() + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + const auto changefeed = TChangefeedDescription("feed", EChangefeedMode::KeysOnly, EChangefeedFormat::Json) + .AddAttribute("key", "value"); + + { + auto result = session.AlterTable("/Root/table", TAlterTableSettings() + .AppendAddChangefeeds(changefeed) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto result = session.DescribeTable("/Root/table").ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + const auto& changefeeds = result.GetTableDescription().GetChangefeedDescriptions(); + UNIT_ASSERT_VALUES_EQUAL(changefeeds.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(changefeeds.at(0), changefeed); + UNIT_ASSERT_VALUES_EQUAL(changefeeds.at(0).GetAttributes(), changefeed.GetAttributes()); + } + } + Y_UNIT_TEST(CreatedAt) { TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig())); auto scheme = NYdb::NScheme::TSchemeClient(kikimr.GetDriver(), TCommonClientSettings().Database("/Root")); @@ -3408,25 +3524,60 @@ Y_UNIT_TEST_SUITE(KqpScheme) { );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +#if 0 // TODO + { // describe table + auto desc = session.DescribeTable(tableName).ExtractValueSync(); + UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); + auto tiering = desc.GetTableDescription().GetTiering(); + UNIT_ASSERT(tiering); + UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering1"); + } +#endif auto query2 = TStringBuilder() << R"( --!syntax_v1 ALTER TABLE `)" << tableName << R"(` SET(TIERING = 'tiering2');)"; result = session.ExecuteSchemeQuery(query2).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + { // describe table + auto desc = session.DescribeTable(tableName).ExtractValueSync(); + UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); + + auto tiering = desc.GetTableDescription().GetTiering(); + UNIT_ASSERT(tiering); + UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering2"); + } + auto query3 = TStringBuilder() << R"( --!syntax_v1 ALTER TABLE `)" << tableName << R"(` RESET (TIERING);)"; result = session.ExecuteSchemeQuery(query3).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + { // describe table + auto desc = session.DescribeTable(tableName).ExtractValueSync(); + UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); + + auto tiering = desc.GetTableDescription().GetTiering(); + UNIT_ASSERT(!tiering); + } + auto query4 = TStringBuilder() << R"( --!syntax_v1 ALTER TABLE `)" << tableName << R"(` SET (TIERING = 'tiering1');)"; result = session.ExecuteSchemeQuery(query4).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + { // describe table + auto desc = session.DescribeTable(tableName).ExtractValueSync(); + UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); + + auto tiering = desc.GetTableDescription().GetTiering(); + UNIT_ASSERT(tiering); + UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering1"); + } + auto query5 = TStringBuilder() << R"( --!syntax_v1 DROP TABLE `)" << tableName << R"(`;)"; diff --git a/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp b/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp index b08192a1a6..db5bfaa6f3 100644 --- a/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_document_api_ut.cpp @@ -115,10 +115,16 @@ Y_UNIT_TEST_SUITE(KqpDocumentApi) { ALTER TABLE `/Root/DocumentApiTest` DROP COLUMN Value; )"; - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_OPERATION)); + const auto tests = TVector<std::pair<TExecSchemeQuerySettings, bool>>{ + {TExecSchemeQuerySettings(), false}, + {TExecSchemeQuerySettings().RequestType("_document_api_request"), true}, + }; + + for (const auto& [settings, success] : tests) { + auto result = session.ExecuteSchemeQuery(query, settings).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_VALUES_EQUAL(result.IsSuccess(), success); + } } Y_UNIT_TEST(RestrictDrop) { diff --git a/ydb/core/mind/address_classification/net_classifier.cpp b/ydb/core/mind/address_classification/net_classifier.cpp index 11c7e6750c..c14992d9e8 100644 --- a/ydb/core/mind/address_classification/net_classifier.cpp +++ b/ydb/core/mind/address_classification/net_classifier.cpp @@ -99,6 +99,21 @@ private: hFunc(TEvConfigsDispatcher::TEvGetConfigResponse, HandleWhileIniting); hFunc(TEvents::TEvWakeup, HandleWhileIniting); hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters); + default: + EnqueueEvent(ev); + break; + } + } + + void EnqueueEvent(TAutoPtr<IEventHandle> &ev) { + EventsQueue.push_back(ev); + } + + void ProcessEnqueuedEvents() { + while (!EventsQueue.empty()) { + TAutoPtr<IEventHandle> &ev = EventsQueue.front(); + TlsActivationContext->Send(ev.Release()); + EventsQueue.pop_front(); } } @@ -224,6 +239,8 @@ private: mon->RegisterActorPage(page, "netclassifier", "NetClassifier", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID); } + ProcessEnqueuedEvents(); + BroadcastClassifierUpdate(); } @@ -395,6 +412,7 @@ private: private: TMaybe<TString> MaybeNetDataFilePath; TMaybe<TInstant> MaybeNetDataFileModTs; + TDeque<TAutoPtr<IEventHandle>> EventsQueue; TLabeledAddressClassifier::TConstPtr LabeledAddressClassifier; TMaybe<TInstant> NetDataUpdateTimestamp; diff --git a/ydb/core/mind/address_classification/net_classifier_ut.cpp b/ydb/core/mind/address_classification/net_classifier_ut.cpp index 67262bc42f..8655a47197 100644 --- a/ydb/core/mind/address_classification/net_classifier_ut.cpp +++ b/ydb/core/mind/address_classification/net_classifier_ut.cpp @@ -97,7 +97,7 @@ Y_UNIT_TEST_SUITE(TNetClassifierTest) { UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1); UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 2); UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::File); const auto prevLagSeconds = AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic()); @@ -130,7 +130,7 @@ Y_UNIT_TEST_SUITE(TNetClassifierTest) { UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1); UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0); - UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1); + UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 2); UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::None); } } diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index 4e8e493dd4..ff4460413c 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -28,7 +28,7 @@ TBlobStorageController::TVSlotInfo::TVSlotInfo(TVSlotId vSlotId, TPDiskInfo *pdi Table::GroupGeneration::Type groupPrevGeneration, Table::GroupGeneration::Type groupGeneration, Table::Category::Type kind, Table::RingIdx::Type ringIdx, Table::FailDomainIdx::Type failDomainIdx, Table::VDiskIdx::Type vDiskIdx, Table::Mood::Type mood, TGroupInfo *group, - TVSlotReadyTimestampQ *vslotReadyTimestampQ, TInstant lastSeenReady) + TVSlotReadyTimestampQ *vslotReadyTimestampQ, TInstant lastSeenReady, TDuration replicationTime) : VSlotId(vSlotId) , PDisk(pdisk) , GroupId(groupId) @@ -40,6 +40,7 @@ TBlobStorageController::TVSlotInfo::TVSlotInfo(TVSlotId vSlotId, TPDiskInfo *pdi , VDiskIdx(vDiskIdx) , Mood(mood) , LastSeenReady(lastSeenReady) + , ReplicationTime(replicationTime) , VSlotReadyTimestampQ(*vslotReadyTimestampQ) { Y_VERIFY(pdisk); diff --git a/ydb/core/mind/bscontroller/config_fit_groups.cpp b/ydb/core/mind/bscontroller/config_fit_groups.cpp index a5ec4df6bb..dd849aa1d3 100644 --- a/ydb/core/mind/bscontroller/config_fit_groups.cpp +++ b/ydb/core/mind/bscontroller/config_fit_groups.cpp @@ -555,7 +555,7 @@ namespace NKikimr { TVSlotInfo *vslotInfo = State.VSlots.ConstructInplaceNewEntry(vslotId, vslotId, pdiskInfo, groupInfo->ID, 0, groupInfo->Generation, StoragePool.VDiskKind, failRealmIdx, failDomainIdx, vdiskIdx, TMood::Normal, groupInfo, &VSlotReadyTimestampQ, - TInstant::Zero()); + TInstant::Zero(), TDuration::Zero()); // mark as uncommitted State.UncommittedVSlots.insert(vslotId); diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index 3f61958f9f..a477c015ec 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -99,6 +99,8 @@ public: TIndirectReferable<TGroupInfo>::TPtr Group; // group to which this VSlot belongs (or nullptr if it doesn't belong to any) THashSet<TVSlotId> Donors; // a set of alive donors for this disk (which are not being deleted) TInstant LastSeenReady; + TInstant LastGotReplicating; + TDuration ReplicationTime; // volatile state mutable NKikimrBlobStorage::TVDiskMetrics Metrics; @@ -127,8 +129,22 @@ public: bool IsReady = false; public: - void SetStatus(NKikimrBlobStorage::EVDiskStatus status, TMonotonic now) { + void SetStatus(NKikimrBlobStorage::EVDiskStatus status, TMonotonic now, TInstant instant) { if (status != Status) { + if (status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // became "replicating" + LastGotReplicating = instant; + } else if (Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // was "replicating" + Y_VERIFY_DEBUG(LastGotReplicating != TInstant::Zero()); + ReplicationTime += instant - LastGotReplicating; + LastGotReplicating = {}; + } + if (status == NKikimrBlobStorage::EVDiskStatus::READY) { + ReplicationTime = TDuration::Zero(); + } + if (IsReady) { + LastSeenReady = instant; + } + Status = status; IsReady = false; if (status == NKikimrBlobStorage::EVDiskStatus::READY) { @@ -176,7 +192,9 @@ public: Table::VDiskIdx, Table::GroupPrevGeneration, Table::Mood, - Table::LastSeenReady + Table::LastSeenReady, + Table::LastGotReplicating, + Table::ReplicationTime > adapter( &TVSlotInfo::Kind, &TVSlotInfo::GroupId, @@ -186,7 +204,9 @@ public: &TVSlotInfo::VDiskIdx, &TVSlotInfo::GroupPrevGeneration, &TVSlotInfo::Mood, - &TVSlotInfo::LastSeenReady + &TVSlotInfo::LastSeenReady, + &TVSlotInfo::LastGotReplicating, + &TVSlotInfo::ReplicationTime ); callback(&adapter); } @@ -196,7 +216,8 @@ public: TVSlotInfo(TVSlotId vSlotId, TPDiskInfo *pdisk, TGroupId groupId, Table::GroupGeneration::Type groupPrevGeneration, Table::GroupGeneration::Type groupGeneration, Table::Category::Type kind, Table::RingIdx::Type ringIdx, Table::FailDomainIdx::Type failDomainIdx, Table::VDiskIdx::Type vDiskIdx, Table::Mood::Type mood, - TGroupInfo *group, TVSlotReadyTimestampQ *vslotReadyTimestampQ, TInstant lastSeenReady); // implemented in bsc.cpp + TGroupInfo *group, TVSlotReadyTimestampQ *vslotReadyTimestampQ, TInstant lastSeenReady, + TDuration replicationTime); // implemented in bsc.cpp // is the slot being deleted (marked as deleted) bool IsBeingDeleted() const { @@ -1796,7 +1817,24 @@ private: ITransaction* CreateTxMigrate(); ITransaction* CreateTxLoadEverything(); ITransaction* CreateTxUpdateSeenOperational(TVector<TGroupId> groups); - ITransaction* CreateTxUpdateLastSeenReady(std::vector<std::pair<TVSlotId, TInstant>> lastSeenReadyQ); + + struct TVDiskAvailabilityTiming { + TVSlotId VSlotId; + TInstant LastSeenReady; + TInstant LastGotReplicating; + TDuration ReplicationTime; + + TVDiskAvailabilityTiming() = default; + TVDiskAvailabilityTiming(const TVDiskAvailabilityTiming&) = default; + + TVDiskAvailabilityTiming(const TVSlotInfo& vslot) + : VSlotId(vslot.VSlotId) + , LastSeenReady(vslot.LastSeenReady) + , LastGotReplicating(vslot.LastGotReplicating) + , ReplicationTime(vslot.ReplicationTime) + {} + }; + ITransaction* CreateTxUpdateLastSeenReady(std::vector<TVDiskAvailabilityTiming> timingQ); void Handle(TEvPrivate::TEvDropDonor::TPtr ev); @@ -2040,7 +2078,7 @@ public: bool VSlotReadyUpdateScheduled = false; void VSlotReadyUpdate() { - std::vector<std::pair<TVSlotId, TInstant>> lastSeenReadyQ; + std::vector<TVDiskAvailabilityTiming> timingQ; Y_VERIFY(VSlotReadyUpdateScheduled); VSlotReadyUpdateScheduled = false; @@ -2054,7 +2092,7 @@ public: if (const TGroupInfo *group = it->second->Group) { groups.insert(const_cast<TGroupInfo*>(group)); it->second->LastSeenReady = TInstant::Zero(); - lastSeenReadyQ.emplace_back(it->second->VSlotId, it->second->LastSeenReady); + timingQ.emplace_back(*it->second); NotReadyVSlotIds.erase(it->second->VSlotId); } ScrubState.UpdateVDiskState(&*it->second); @@ -2063,8 +2101,8 @@ public: group->CalculateGroupStatus(); } ScheduleVSlotReadyUpdate(); - if (!lastSeenReadyQ.empty()) { - Execute(CreateTxUpdateLastSeenReady(std::move(lastSeenReadyQ))); + if (!timingQ.empty()) { + Execute(CreateTxUpdateLastSeenReady(std::move(timingQ))); } } @@ -2077,12 +2115,28 @@ public: void VSlotNotReadyHistogramUpdate() { const TInstant now = TActivationContext::Now(); auto& histo = TabletCounters->Percentile()[NBlobStorageController::COUNTER_NUM_NOT_READY_VDISKS]; + auto& histoReplROT = TabletCounters->Percentile()[NBlobStorageController::COUNTER_NUM_REPLICATING_VDISKS_ROT]; + auto& histoReplOther = TabletCounters->Percentile()[NBlobStorageController::COUNTER_NUM_REPLICATING_VDISKS_OTHER]; histo.Clear(); + histoReplROT.Clear(); + histoReplOther.Clear(); for (const TVSlotId vslotId : NotReadyVSlotIds) { if (const TVSlotInfo *slot = FindVSlot(vslotId)) { Y_VERIFY(slot->LastSeenReady != TInstant::Zero()); const TDuration passed = now - slot->LastSeenReady; histo.IncrementFor(passed.Seconds()); + + TDuration timeBeingReplicating = slot->ReplicationTime; + if (slot->Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { + timeBeingReplicating += now - slot->LastGotReplicating; + } + + if (timeBeingReplicating != TDuration::Zero()) { + auto& hist = slot->PDisk->Kind.Type() == NPDisk::DEVICE_TYPE_ROT + ? histoReplROT + : histoReplOther; + hist.IncrementFor(timeBeingReplicating.Seconds()); + } } else { Y_VERIFY_DEBUG(false); } diff --git a/ydb/core/mind/bscontroller/load_everything.cpp b/ydb/core/mind/bscontroller/load_everything.cpp index 14c8637aec..27ede3359d 100644 --- a/ydb/core/mind/bscontroller/load_everything.cpp +++ b/ydb/core/mind/bscontroller/load_everything.cpp @@ -363,7 +363,8 @@ public: auto& x = Self->AddVSlot(vslotId, pdisk, groupId, slot.GetValueOrDefault<T::GroupPrevGeneration>(), slot.GetValue<T::GroupGeneration>(), slot.GetValue<T::Category>(), slot.GetValue<T::RingIdx>(), slot.GetValue<T::FailDomainIdx>(), slot.GetValue<T::VDiskIdx>(), slot.GetValueOrDefault<T::Mood>(), - Self->FindGroup(groupId), &Self->VSlotReadyTimestampQ, slot.GetValue<T::LastSeenReady>()); + Self->FindGroup(groupId), &Self->VSlotReadyTimestampQ, slot.GetValue<T::LastSeenReady>(), + slot.GetValue<T::ReplicationTime>()); if (x.LastSeenReady != TInstant::Zero()) { Self->NotReadyVSlotIds.insert(x.VSlotId); } diff --git a/ydb/core/mind/bscontroller/monitoring.cpp b/ydb/core/mind/bscontroller/monitoring.cpp index 6c84c19929..f67d0a9f6a 100644 --- a/ydb/core/mind/bscontroller/monitoring.cpp +++ b/ydb/core/mind/bscontroller/monitoring.cpp @@ -1252,6 +1252,7 @@ void TBlobStorageController::RenderVSlotTable(IOutputStream& out, std::function< TABLEH() { out << "Status"; } TABLEH() { out << "IsReady"; } TABLEH() { out << "LastSeenReady"; } + TABLEH() { out << "ReplicationTime"; } TABLEH() { out << "Donors"; } TABLEH() { out << "Data Size"; } @@ -1291,6 +1292,13 @@ void TBlobStorageController::RenderVSlotRow(IOutputStream& out, const TVSlotInfo } } TABLED() { + TDuration time = vslot.ReplicationTime; + if (vslot.Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { + time += TActivationContext::Now() - vslot.LastGotReplicating; + } + out << time; + } + TABLED() { if (vslot.Mood == TMood::Donor) { const auto *x = FindAcceptor(vslot); out << "<strong>donor for <a href='#" << x->GetVDiskId() << "'>" << x->VSlotId << "</a></strong>"; diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index 06c54413c3..adb3f0745b 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -482,7 +482,7 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId) { const TInstant now = TActivationContext::Now(); const TMonotonic mono = TActivationContext::Monotonic(); - std::vector<std::pair<TVSlotId, TInstant>> lastSeenReadyQ; + std::vector<TVDiskAvailabilityTiming> timingQ; for (auto it = PDisks.lower_bound(TPDiskId::MinForNode(nodeId)); it != PDisks.end() && it->first.NodeId == nodeId; ++it) { it->second->UpdateOperational(false); SysViewChangedPDisks.insert(it->first); @@ -492,11 +492,10 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId) { for (auto it = VSlots.lower_bound(startingId); it != VSlots.end() && it->first.NodeId == nodeId; ++it) { if (const TGroupInfo *group = it->second->Group) { if (it->second->IsReady) { - it->second->LastSeenReady = now; - lastSeenReadyQ.emplace_back(it->second->VSlotId, now); NotReadyVSlotIds.insert(it->second->VSlotId); } - it->second->SetStatus(NKikimrBlobStorage::EVDiskStatus::ERROR, mono); + it->second->SetStatus(NKikimrBlobStorage::EVDiskStatus::ERROR, mono, now); + timingQ.emplace_back(*it->second); sh->VDiskStatusUpdate.emplace_back(it->second->GetVDiskId(), it->second->Status); ScrubState.UpdateVDiskState(&*it->second); } @@ -509,8 +508,8 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId) { } ScrubState.OnNodeDisconnected(nodeId); EraseKnownDrivesOnDisconnected(&node); - if (!lastSeenReadyQ.empty()) { - Execute(CreateTxUpdateLastSeenReady(std::move(lastSeenReadyQ))); + if (!timingQ.empty()) { + Execute(CreateTxUpdateLastSeenReady(std::move(timingQ))); } for (TGroupId groupId : std::exchange(node.GroupsRequested, {})) { GroupToNode.erase(std::make_tuple(groupId, nodeId)); diff --git a/ydb/core/mind/bscontroller/scheme.h b/ydb/core/mind/bscontroller/scheme.h index e97b0df955..f6d0957a2b 100644 --- a/ydb/core/mind/bscontroller/scheme.h +++ b/ydb/core/mind/bscontroller/scheme.h @@ -121,9 +121,25 @@ struct Schema : NIceDb::Schema { struct GroupPrevGeneration : Column<10, Group::Generation::ColumnType> { static constexpr ui32 Default = 0; }; struct Mood : Column<11, NScheme::NTypeIds::Uint32> { static constexpr ui32 Default = 0; }; struct LastSeenReady : Column<12, NScheme::NTypeIds::Uint64> { using Type = TInstant; static constexpr Type Default = TInstant::Zero(); }; + struct LastGotReplicating : Column<13, NScheme::NTypeIds::Uint64> { using Type = TInstant; static constexpr Type Default = TInstant::Zero(); }; + struct ReplicationTime : Column<14, NScheme::NTypeIds::Uint64> { using Type = TDuration; static constexpr Type Default = TDuration::Zero(); }; using TKey = TableKey<NodeID, PDiskID, VSlotID>; // order is important - using TColumns = TableColumns<NodeID, PDiskID, VSlotID, Category, GroupID, GroupGeneration, RingIdx, FailDomainIdx, VDiskIdx, GroupPrevGeneration, Mood, LastSeenReady>; + using TColumns = TableColumns< + NodeID, + PDiskID, + VSlotID, + Category, + GroupID, + GroupGeneration, + RingIdx, + FailDomainIdx, + VDiskIdx, + GroupPrevGeneration, + Mood, + LastSeenReady, + LastGotReplicating, + ReplicationTime>; }; struct VDiskMetrics : Table<6> { diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index f33a7b3024..e4befa1540 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -631,7 +631,7 @@ namespace NKikimr::NBsController { THashSet<TGroupInfo*> groups; const TInstant now = TActivationContext::Now(); const TMonotonic mono = TActivationContext::Monotonic(); - std::vector<std::pair<TVSlotId, TInstant>> lastSeenReadyQ; + std::vector<TVDiskAvailabilityTiming> timingQ; std::unique_ptr<TEvPrivate::TEvDropDonor> dropDonorEv; @@ -644,14 +644,15 @@ namespace NKikimr::NBsController { const bool was = slot->IsOperational(); if (const TGroupInfo *group = slot->Group) { const bool wasReady = slot->IsReady; - slot->SetStatus(m.GetStatus(), mono); - if (slot->IsReady != wasReady) { - ScrubState.UpdateVDiskState(slot); - if (wasReady) { - slot->LastSeenReady = now; - lastSeenReadyQ.emplace_back(slot->VSlotId, now); - NotReadyVSlotIds.insert(slot->VSlotId); + if (slot->Status != m.GetStatus()) { + slot->SetStatus(m.GetStatus(), mono, now); + if (slot->IsReady != wasReady) { + ScrubState.UpdateVDiskState(slot); + if (wasReady) { + NotReadyVSlotIds.insert(slot->VSlotId); + } } + timingQ.emplace_back(*slot); } ev->VDiskStatusUpdate.emplace_back(vdiskId, m.GetStatus()); if (!was && slot->IsOperational() && !group->SeenOperational) { @@ -695,8 +696,8 @@ namespace NKikimr::NBsController { Execute(CreateTxUpdateSeenOperational(std::move(groupIds))); } - if (!lastSeenReadyQ.empty()) { - Execute(CreateTxUpdateLastSeenReady(std::move(lastSeenReadyQ))); + if (!timingQ.empty()) { + Execute(CreateTxUpdateLastSeenReady(std::move(timingQ))); } ScheduleVSlotReadyUpdate(); diff --git a/ydb/core/mind/bscontroller/update_last_seen_ready.cpp b/ydb/core/mind/bscontroller/update_last_seen_ready.cpp index daf18702d8..c50298142a 100644 --- a/ydb/core/mind/bscontroller/update_last_seen_ready.cpp +++ b/ydb/core/mind/bscontroller/update_last_seen_ready.cpp @@ -4,20 +4,31 @@ namespace NKikimr { namespace NBsController { class TBlobStorageController::TTxUpdateLastSeenReady : public TTransactionBase<TBlobStorageController> { - std::vector<std::pair<TVSlotId, TInstant>> LastSeenReadyQ; + std::vector<TVDiskAvailabilityTiming> TimingQ; public: - TTxUpdateLastSeenReady(std::vector<std::pair<TVSlotId, TInstant>> lastSeenReadyQ, TBlobStorageController *controller) + TTxUpdateLastSeenReady(std::vector<TVDiskAvailabilityTiming> timingQ, TBlobStorageController *controller) : TBase(controller) - , LastSeenReadyQ(std::move(lastSeenReadyQ)) + , TimingQ(std::move(timingQ)) {} TTxType GetTxType() const override { return NBlobStorageController::TXTYPE_UPDATE_LAST_SEEN_READY; } bool Execute(TTransactionContext &txc, const TActorContext&) override { NIceDb::TNiceDb db(txc.DB); - for (const auto& [vslotId, lastSeenReady] : LastSeenReadyQ) { - db.Table<Schema::VSlot>().Key(vslotId.GetKey()).Update<Schema::VSlot::LastSeenReady>(lastSeenReady); + for (const auto& item : TimingQ) { + auto row = db.Table<Schema::VSlot>().Key(item.VSlotId.GetKey()); + +#define UPDATE_CELL(CELL) \ + if (item.CELL != Schema::VSlot::CELL::Default) { \ + row.Update<Schema::VSlot::CELL>(item.CELL); \ + } else { \ + row.UpdateToNull<Schema::VSlot::CELL>(); \ + } + + UPDATE_CELL(LastSeenReady); + UPDATE_CELL(LastGotReplicating); + UPDATE_CELL(ReplicationTime); } return true; } @@ -25,8 +36,8 @@ public: void Complete(const TActorContext&) override {} }; -ITransaction* TBlobStorageController::CreateTxUpdateLastSeenReady(std::vector<std::pair<TVSlotId, TInstant>> lastSeenReadyQ) { - return new TTxUpdateLastSeenReady(std::move(lastSeenReadyQ), this); +ITransaction* TBlobStorageController::CreateTxUpdateLastSeenReady(std::vector<TVDiskAvailabilityTiming> timingQ) { + return new TTxUpdateLastSeenReady(std::move(timingQ), this); } } diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index f4d54fddfe..de72e90256 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -1934,6 +1934,47 @@ Y_UNIT_TEST_SUITE(THiveTest) { } } + Y_UNIT_TEST(TestUpdateChannelValues) { + TTestBasicRuntime runtime(1, false); + Setup(runtime, true, 2); + + const ui64 hiveTablet = MakeDefaultHiveID(0); + const ui64 testerTablet = MakeDefaultHiveID(1); + const TActorId sender = runtime.AllocateEdgeActor(); + CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive); + CreateLocal(runtime, 0); + + TTabletTypes::EType tabletType = TTabletTypes::Dummy; + TChannelsBindings channels = BINDED_CHANNELS; + for (auto& bind : channels) { + bind.SetSize(1000); + } + TAutoPtr<TEvHive::TEvCreateTablet> createTablet(new TEvHive::TEvCreateTablet(testerTablet, 0, tabletType, channels)); + ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, createTablet, 0, true); + + MakeSureTabletIsUp(runtime, tabletId, 0); + + for (auto& bind : channels) { + bind.SetSize(1001); + } + channels[0].SetStoragePoolName("def2"); + channels[1].SetStoragePoolName("def1"); + TAutoPtr<TEvHive::TEvCreateTablet> updateTablet(new TEvHive::TEvCreateTablet(testerTablet, 0, tabletType, channels)); + tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, updateTablet, 0, true); + + runtime.SendToPipe(hiveTablet, sender, new TEvHive::TEvRequestHiveStorageStats()); + TAutoPtr<IEventHandle> handle; + TEvHive::TEvResponseHiveStorageStats* storageStats = runtime.GrabEdgeEventRethrow<TEvHive::TEvResponseHiveStorageStats>(handle); + + for (const auto& pool : storageStats->Record.GetPools()) { + for (const auto& group : pool.GetGroups()) { + if (group.GetAcquiredSize() != 0) { + UNIT_ASSERT_VALUES_EQUAL(group.GetAcquiredSize(), 1001); + } + } + } + } + Y_UNIT_TEST(TestDeleteTablet) { TTestBasicRuntime runtime(1, false); Setup(runtime, true); diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index 3afccc647f..15fe81b935 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -746,7 +746,8 @@ public: TTabletTypes::PersQueueReadBalancer, TTabletTypes::NodeBroker, TTabletTypes::TestShard, - TTabletTypes::BlobDepot}) { + TTabletTypes::BlobDepot, + TTabletTypes::ColumnShard}) { if (shortType == LongToShortTabletName(TTabletTypes::TypeToStr(tabletType))) { return tabletType; } @@ -1078,7 +1079,8 @@ public: TTabletTypes::PersQueueReadBalancer, TTabletTypes::NodeBroker, TTabletTypes::TestShard, - TTabletTypes::BlobDepot}) { + TTabletTypes::BlobDepot, + TTabletTypes::ColumnShard}) { const TVector<i64>& allowedMetrics = Self->GetTabletTypeAllowedMetricIds(tabletType); out << "<tr>" "<td>" << LongToShortTabletName(TTabletTypes::TypeToStr(tabletType)) << "</td>"; diff --git a/ydb/core/mind/hive/tx__create_tablet.cpp b/ydb/core/mind/hive/tx__create_tablet.cpp index 426acd37f3..f38fde5e05 100644 --- a/ydb/core/mind/hive/tx__create_tablet.cpp +++ b/ydb/core/mind/hive/tx__create_tablet.cpp @@ -117,6 +117,7 @@ public: db.Table<Schema::TabletChannel>().Key(TabletId, channelId).Update<Schema::TabletChannel::Binding>(BoundChannels[channelId]); db.Table<Schema::TabletChannel>().Key(TabletId, channelId).Update<Schema::TabletChannel::NeedNewGroup>(true); newChannels.set(channelId); + tablet.ReleaseAllocationUnit(channelId); } } diff --git a/ydb/core/mind/hive/tx__update_tablet_groups.cpp b/ydb/core/mind/hive/tx__update_tablet_groups.cpp index abf4244984..36cb03f20b 100644 --- a/ydb/core/mind/hive/tx__update_tablet_groups.cpp +++ b/ydb/core/mind/hive/tx__update_tablet_groups.cpp @@ -136,7 +136,7 @@ public: channel = &tabletChannels[channelId]; Y_VERIFY(channel->Channel == channelId); if (!tablet->ReleaseAllocationUnit(channelId)) { - BLOG_ERROR("Failed to release AU for tablet " << tablet->Id << " channel " << channelId); + BLOG_W("Failed to release AU for tablet " << tablet->Id << " channel " << channelId); } } else { // increasing number of tablet channels diff --git a/ydb/core/mind/node_broker.cpp b/ydb/core/mind/node_broker.cpp index ca7aa9cbc5..829717d598 100644 --- a/ydb/core/mind/node_broker.cpp +++ b/ydb/core/mind/node_broker.cpp @@ -132,6 +132,8 @@ void TNodeBroker::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TNodeBroker::Cleanup"); + NConsole::UnsubscribeViaConfigDispatcher(ctx, ctx.SelfID); + TxProcessor->Clear(); } @@ -392,14 +394,8 @@ void TNodeBroker::AddNodeToEpochCache(const TNodeInfo &node) void TNodeBroker::SubscribeForConfigUpdates(const TActorContext &ctx) { - if (ConfigSubscriptionId) - return; - ui32 item = (ui32)NKikimrConsole::TConfigItem::NodeBrokerConfigItem; - ctx.Register(NConsole::CreateConfigSubscriber(TabletID(), - {item}, - "", - ctx.SelfID)); + NConsole::SubscribeViaConfigDispatcher(ctx, {item}, ctx.SelfID); } void TNodeBroker::ProcessTx(ITransaction *tx, @@ -762,7 +758,14 @@ void TNodeBroker::DbUpdateNodeLocation(const TNodeInfo &node, void TNodeBroker::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx) { - ProcessTx(CreateTxUpdateConfig(ev), ctx); + if (ev->Get()->Record.HasLocal() && ev->Get()->Record.GetLocal()) { + ProcessTx(CreateTxUpdateConfig(ev), ctx); + } else { + // ignore and immediately ack messages from old persistent console subscriptions + auto response = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(); + response->Record.MutableConfigId()->CopyFrom(ev->Get()->Record.GetConfigId()); + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } } void TNodeBroker::Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev, diff --git a/ydb/core/mind/node_broker__update_config.cpp b/ydb/core/mind/node_broker__update_config.cpp index d741cefae5..a038625571 100644 --- a/ydb/core/mind/node_broker__update_config.cpp +++ b/ydb/core/mind/node_broker__update_config.cpp @@ -10,7 +10,7 @@ public: TEvConsole::TEvConfigNotificationRequest::TPtr notification) : TBase(self) , Notification(std::move(notification)) - , Config(Notification->Get()->Record.GetConfig().GetNodeBrokerConfig()) + , Config(Notification->Get()->GetConfig().GetNodeBrokerConfig()) , Modify(false) { } @@ -31,12 +31,8 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, "TTxUpdateConfig Execute " << rec.ShortDebugString()); - if (rec.GetSubscriptionId() != Self->ConfigSubscriptionId) { - LOG_ERROR_S(ctx, NKikimrServices::NODE_BROKER, - "Config subscription id mismatch (" << rec.GetSubscriptionId() - << " vs expected " << Self->ConfigSubscriptionId << ")"); - return false; - } + if (!google::protobuf::util::MessageDifferencer::Equals(Config, Self->Config)) + Modify = true; auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); Response = new IEventHandle(Notification->Sender, Self->SelfId(), resp.Release(), @@ -52,6 +48,9 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER, "TTxUpdateConfig Execute " << rec.ShortDebugString()); + if (!google::protobuf::util::MessageDifferencer::Equals(Config, Self->Config)) + Modify = true; + auto resp = MakeHolder<TEvNodeBroker::TEvSetConfigResponse>(); resp->Record.MutableStatus()->SetCode(NKikimrNodeBroker::TStatus::OK); Response = new IEventHandle(Request->Sender, Self->SelfId(), resp.Release(), @@ -69,9 +68,8 @@ public: if (Request && !ProcessRequest(ctx)) return true; - Self->DbUpdateConfig(Config, txc); - - Modify = true; + if (Modify) + Self->DbUpdateConfig(Config, txc); return true; } diff --git a/ydb/core/mind/node_broker_impl.h b/ydb/core/mind/node_broker_impl.h index 212e84c5cf..cbdb461832 100644 --- a/ydb/core/mind/node_broker_impl.h +++ b/ydb/core/mind/node_broker_impl.h @@ -4,6 +4,7 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/console.h> +#include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/tx_processor.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> @@ -179,6 +180,8 @@ private: HFuncTraced(TEvPrivate::TEvUpdateEpoch, Handle); IgnoreFunc(TEvTabletPipe::TEvServerConnected); IgnoreFunc(TEvTabletPipe::TEvServerDisconnected); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { diff --git a/ydb/core/mind/table_adapter.h b/ydb/core/mind/table_adapter.h index c782dc32f4..e2c27e98d2 100644 --- a/ydb/core/mind/table_adapter.h +++ b/ydb/core/mind/table_adapter.h @@ -117,6 +117,10 @@ namespace NKikimr { to.ConstructInPlace(from.GetValue()); } + inline void Cast(const TDuration& from, TMaybe<ui64>& to) { + to.ConstructInPlace(from.GetValue()); + } + template<typename TRow, typename TColumn> struct TCell { typename TColumn::Type TRow::*CellPtr = nullptr; diff --git a/ydb/core/mind/tenant_slot_broker.cpp b/ydb/core/mind/tenant_slot_broker.cpp index 65f7a86328..1b9e5e7f6f 100644 --- a/ydb/core/mind/tenant_slot_broker.cpp +++ b/ydb/core/mind/tenant_slot_broker.cpp @@ -415,10 +415,7 @@ void TTenantSlotBroker::OnActivateExecutor(const TActorContext &ctx) tabletCounters->RemoveSubgroup("type", "TENANT_SLOT_BROKER"); Counters = new TCounters(tabletCounters->GetSubgroup("type", "TENANT_SLOT_BROKER")); - ctx.Register(NConsole::CreateConfigSubscriber(TabletID(), - {(ui32)NKikimrConsole::TConfigItem::TenantSlotBrokerConfigItem}, - "", - ctx.SelfID)); + NConsole::SubscribeViaConfigDispatcher(ctx, {(ui32)NKikimrConsole::TConfigItem::TenantSlotBrokerConfigItem}, ctx.SelfID); ProcessTx(CreateTxInitScheme(), ctx); } @@ -615,6 +612,8 @@ bool TTenantSlotBroker::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, void TTenantSlotBroker::Cleanup(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::TENANT_SLOT_BROKER, "Cleanup"); + + NConsole::UnsubscribeViaConfigDispatcher(ctx, ctx.SelfID); } void TTenantSlotBroker::Die(const TActorContext &ctx) @@ -625,6 +624,7 @@ void TTenantSlotBroker::Die(const TActorContext &ctx) void TTenantSlotBroker::LoadConfigFromProto(const NKikimrTenantSlotBroker::TConfig &config) { + Config = config; PendingTimeout = TDuration::MicroSeconds(config.GetPendingSlotTimeout()); } @@ -1624,7 +1624,14 @@ void TTenantSlotBroker::ProcessNextTx(const TActorContext &ctx) void TTenantSlotBroker::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx) { - ProcessTx(CreateTxUpdateConfig(ev), ctx); + if (ev->Get()->Record.HasLocal() && ev->Get()->Record.GetLocal()) { + ProcessTx(CreateTxUpdateConfig(ev), ctx); + } else { + // ignore and immediately ack messages from old persistent console subscriptions + auto response = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(); + response->Record.MutableConfigId()->CopyFrom(ev->Get()->Record.GetConfigId()); + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } } void TTenantSlotBroker::Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev, diff --git a/ydb/core/mind/tenant_slot_broker__update_config.cpp b/ydb/core/mind/tenant_slot_broker__update_config.cpp index 360b28d9c1..c9f77a0dfc 100644 --- a/ydb/core/mind/tenant_slot_broker__update_config.cpp +++ b/ydb/core/mind/tenant_slot_broker__update_config.cpp @@ -19,20 +19,21 @@ public: LOG_DEBUG_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, "TTxUpdateConfig Execute " << rec.ShortDebugString()); - if (rec.GetSubscriptionId() != Self->ConfigSubscriptionId) { - LOG_ERROR_S(ctx, NKikimrServices::TENANT_SLOT_BROKER, - "Config subscription id mismatch (" << rec.GetSubscriptionId() - << " vs expected " << Self->ConfigSubscriptionId << ")"); - return true; - } - NIceDb::TNiceDb db(txc.DB); - TString config; - Y_PROTOBUF_SUPPRESS_NODISCARD rec.GetConfig().GetTenantSlotBrokerConfig().SerializeToString(&config); - db.Table<Schema::Config>().Key(ConfigKey_Config) - .Update(NIceDb::TUpdate<Schema::Config::Value>(config)); - Modify = true; + const auto &config = Event->Get()->GetConfig().GetTenantSlotBrokerConfig(); + + if (!google::protobuf::util::MessageDifferencer::Equals(config, Self->Config)) { + TString serializedConfig = config.SerializeAsString(); + db.Table<Schema::Config>().Key(ConfigKey_Config) + .Update(NIceDb::TUpdate<Schema::Config::Value>(serializedConfig)); + + Modify = true; + } + + auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); + Response = new IEventHandle(Event->Sender, Self->SelfId(), resp.Release(), + 0, Event->Cookie); return true; } @@ -44,16 +45,17 @@ public: if (Modify) { auto &rec = Event->Get()->Record; Self->LoadConfigFromProto(rec.GetConfig().GetTenantSlotBrokerConfig()); - - auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec); - ctx.Send(Event->Sender, resp.Release(), 0, Event->Cookie); } + if (Response) + ctx.Send(Response); + Self->TxCompleted(this, ctx); } private: TEvConsole::TEvConfigNotificationRequest::TPtr Event; + TAutoPtr<IEventHandle> Response; bool Modify; }; diff --git a/ydb/core/mind/tenant_slot_broker_impl.h b/ydb/core/mind/tenant_slot_broker_impl.h index ef08bb73b3..788e0d5f95 100644 --- a/ydb/core/mind/tenant_slot_broker_impl.h +++ b/ydb/core/mind/tenant_slot_broker_impl.h @@ -6,6 +6,7 @@ #include <ydb/core/base/location.h> #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/console.h> +#include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/mind/tenant_pool.h> #include <ydb/core/protos/tenant_slot_broker.pb.h> @@ -1121,6 +1122,8 @@ private: HFuncTraced(TEvTenantSlotBroker::TEvGetTenantState, Handle); HFuncTraced(TEvTenantSlotBroker::TEvListTenants, Handle); HFuncTraced(TEvTenantSlotBroker::TEvRegisterPool, Handle); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse); + IgnoreFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse); default: if (!HandleDefaultEvents(ev, ctx)) { @@ -1151,6 +1154,7 @@ public: private: TDeque<TAutoPtr<IEventHandle>> InitQueue; + NKikimrTenantSlotBroker::TConfig Config; TDuration PendingTimeout; ui64 RequestId; ui32 DomainId; diff --git a/ydb/core/mind/tenant_ut_pool.cpp b/ydb/core/mind/tenant_ut_pool.cpp index bd3dbc7fc0..2abd1984cd 100644 --- a/ydb/core/mind/tenant_ut_pool.cpp +++ b/ydb/core/mind/tenant_ut_pool.cpp @@ -135,7 +135,7 @@ void ChangeMonitoringConfig(TTenantTestRuntime &runtime, TDispatchOptions options; options.FinalEvents.emplace_back - (TIsConfigNotificationProcessed(3 * runtime.GetNodeCount(), + (TIsConfigNotificationProcessed(2 * runtime.GetNodeCount(), 2 * waitForPoolStatus * runtime.GetNodeCount())); runtime.DispatchEvents(options); } diff --git a/ydb/core/mon/mon_impl.h b/ydb/core/mon/mon_impl.h index ab56828c76..dc54b7c35f 100644 --- a/ydb/core/mon/mon_impl.h +++ b/ydb/core/mon/mon_impl.h @@ -245,7 +245,18 @@ public: void Output(NMonitoring::IMonHttpRequest& request) override { IOutputStream& out = request.Output(); - out << HTTPOKHTML; + out << "HTTP/1.1 200 Ok\r\n" + << "Content-Type: text/html\r\n" + << "Connection: Close\r\n"; + TString origin = TString(request.GetHeader("Origin")); + if (origin.empty()) { + origin = "*"; + } + out << "Access-Control-Allow-Origin: " << origin << "\r\n" + << "Access-Control-Allow-Credentials: true\r\n" + << "Access-Control-Allow-Headers: Content-Type,Authorization,Origin,Accept\r\n" + << "Access-Control-Allow-Methods: OPTIONS, GET, POST\r\n"; + out << "\r\n"; out << "<!DOCTYPE html>\n"; out << "<html>"; diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 1d6f2671fa..c1ed27c836 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -1078,9 +1078,23 @@ ui64 TPartition::MeteringDataSize(const TActorContext& ctx) const { return size; } +ui64 TPartition::ReserveSize() const { + return TopicPartitionReserveSize(Config); +} + +ui64 TPartition::StorageSize(const TActorContext& ctx) const { + return std::max<ui64>(MeteringDataSize(ctx), ReserveSize()); +} + +ui64 TPartition::UsedReserveSize(const TActorContext& ctx) const { + return std::min<ui64>(MeteringDataSize(ctx), ReserveSize()); +} + + ui64 TPartition::GetUsedStorage(const TActorContext& ctx) { - auto duration = ctx.Now() - LastUsedStorageMeterTimestamp; - LastUsedStorageMeterTimestamp = ctx.Now(); + const auto now = ctx.Now(); + const auto duration = now - LastUsedStorageMeterTimestamp; + LastUsedStorageMeterTimestamp = now; ui64 size = MeteringDataSize(ctx); return size * duration.MilliSeconds() / 1000 / 1_MB; // mb*seconds } @@ -1135,8 +1149,8 @@ void TPartition::HandleWakeup(const TActorContext& ctx) { } if (haveChanges) { - WriteCycleStartTime = ctx.Now(); - WriteStartTime = ctx.Now(); + WriteCycleStartTime = now; + WriteStartTime = now; TopicQuotaWaitTimeForCurrentBlob = TDuration::Zero(); WritesTotal.Inc(); Become(&TThis::StateWrite); @@ -2935,7 +2949,7 @@ void TPartition::OnReadRequestFinished(TReadInfo&& info, ui64 answerSize) { auto userInfo = UsersInfoStorage.GetIfExists(info.User); Y_VERIFY(userInfo); - if (Config.GetMeteringMode() == NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS) { + if (Config.GetMeteringMode() != NKikimrPQ::TPQTabletConfig::METERING_MODE_RESERVED_CAPACITY) { return; } @@ -3547,11 +3561,25 @@ bool TPartition::UpdateCounters(const TActorContext& ctx) { } } - ui64 partSize = BodySize + Head.PackedSize; - if (partSize != PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Get()) { + ui64 storageSize = StorageSize(ctx); + if (storageSize != PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Get()) { haveChanges = true; - PartitionCountersLabeled->GetCounters()[METRIC_MAX_PART_SIZE].Set(partSize); - PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Set(partSize); + PartitionCountersLabeled->GetCounters()[METRIC_MAX_PART_SIZE].Set(storageSize); + PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Set(storageSize); + } + + if (NKikimrPQ::TPQTabletConfig::METERING_MODE_RESERVED_CAPACITY == Config.GetMeteringMode()) { + ui64 reserveSize = ReserveSize(); + if (reserveSize != PartitionCountersLabeled->GetCounters()[METRIC_RESERVE_LIMIT_BYTES].Get()) { + haveChanges = true; + PartitionCountersLabeled->GetCounters()[METRIC_RESERVE_LIMIT_BYTES].Set(reserveSize); + } + + ui64 reserveUsed = UsedReserveSize(ctx); + if (reserveUsed != PartitionCountersLabeled->GetCounters()[METRIC_RESERVE_USED_BYTES].Get()) { + haveChanges = true; + PartitionCountersLabeled->GetCounters()[METRIC_RESERVE_USED_BYTES].Set(reserveUsed); + } } ui64 ts = (WriteTimestamp.MilliSeconds() < MIN_TIMESTAMP_MS) ? Max<i64>() : WriteTimestamp.MilliSeconds(); @@ -5654,7 +5682,7 @@ void TPartition::Handle(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& c } size_t TPartition::GetQuotaRequestSize(const TEvKeyValue::TEvRequest& request) { - if (Config.GetMeteringMode() == NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS) { + if (Config.GetMeteringMode() != NKikimrPQ::TPQTabletConfig::METERING_MODE_RESERVED_CAPACITY) { return 0; } if (AppData()->PQConfig.GetQuotingConfig().GetTopicWriteQuotaEntityToLimit() == diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index fca2db03c2..69faa82643 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -309,14 +309,9 @@ public: } ui64 MeteringDataSize(const TActorContext& ctx) const; - - ui64 UsedReserveSize(const TActorContext& ctx) const { - return std::min<ui64>(MeteringDataSize(ctx), ReserveSize()); - } - - ui64 ReserveSize() const { - return TopicPartitionReserveSize(Config); - } + ui64 ReserveSize() const; + ui64 StorageSize(const TActorContext& ctx) const; + ui64 UsedReserveSize(const TActorContext& ctx) const; //Bootstrap sends kvRead diff --git a/ydb/core/persqueue/ut/common/pq_ut_common.cpp b/ydb/core/persqueue/ut/common/pq_ut_common.cpp index 4d936f51f3..5d49a55b32 100644 --- a/ydb/core/persqueue/ut/common/pq_ut_common.cpp +++ b/ydb/core/persqueue/ut/common/pq_ut_common.cpp @@ -67,6 +67,7 @@ void PQTabletPrepare(const TTabletPreparationParameters& parameters, tabletConfig->SetLocalDC(parameters.localDC); tabletConfig->AddReadRules("user"); tabletConfig->AddReadFromTimestampsMs(parameters.readFromTimestampsMs); + tabletConfig->SetMeteringMode(parameters.meteringMode); auto config = tabletConfig->MutablePartitionConfig(); config->SetMaxCountInPartition(parameters.maxCountInPartition); config->SetMaxSizeInPartition(parameters.maxSizeInPartition); diff --git a/ydb/core/persqueue/ut/common/pq_ut_common.h b/ydb/core/persqueue/ut/common/pq_ut_common.h index d40f5ad05e..2444703838 100644 --- a/ydb/core/persqueue/ut/common/pq_ut_common.h +++ b/ydb/core/persqueue/ut/common/pq_ut_common.h @@ -221,6 +221,7 @@ struct TTabletPreparationParameters { TString databaseId{"PQ"}; TString databasePath{"/Root/PQ"}; TString account{"federationAccount"}; + ::NKikimrPQ::TPQTabletConfig_EMeteringMode meteringMode = NKikimrPQ::TPQTabletConfig::METERING_MODE_RESERVED_CAPACITY; }; void PQTabletPrepare( const TTabletPreparationParameters& parameters, diff --git a/ydb/core/persqueue/ut/counters_ut.cpp b/ydb/core/persqueue/ut/counters_ut.cpp index 8201fa5998..804e340c80 100644 --- a/ydb/core/persqueue/ut/counters_ut.cpp +++ b/ydb/core/persqueue/ut/counters_ut.cpp @@ -147,6 +147,9 @@ void CompareJsons(const TString& inputStr, const TString& referenceStr) { NJson::TJsonValue inputJson; UNIT_ASSERT(NJson::ReadJsonTree(TStringBuf(inputStr), &inputJson)); + Cerr << "Expected: " << referenceStr << Endl; + Cerr << "Result: " << inputStr << Endl; + // Run time of test differs as well as counters below. // We set it to 5000 and then compare with reference string. auto getByPath = [](const NJson::TJsonValue& msg, TStringBuf path) { @@ -254,7 +257,7 @@ Y_UNIT_TEST(PartitionFirstClass) { return TTestActorRuntime::DefaultObserverFunc(runtime, event); }); - PQTabletPrepare({}, {{"client", true}}, tc); + PQTabletPrepare({.deleteTime=3600, .meteringMode = NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS}, {{"client", true}}, tc); TFakeSchemeShardState::TPtr state{new TFakeSchemeShardState()}; ui64 ssId = 325; BootFakeSchemeShard(*tc.Runtime, ssId, state); diff --git a/ydb/core/persqueue/ut/resources/counters_labeled.json b/ydb/core/persqueue/ut/resources/counters_labeled.json index e4bc14b3dd..85804aa58b 100644 --- a/ydb/core/persqueue/ut/resources/counters_labeled.json +++ b/ydb/core/persqueue/ut/resources/counters_labeled.json @@ -1 +1,2302 @@ -{"sensors":[{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MessageLagByCommitted"},"value":30},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionMaxReadQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesAvailAvgMin"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesAvailAvgSec"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesQuota"},"value":1000000000},{"kind":"RATE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadOffsetRewindSum"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadTimeLagMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SizeLagByCommitted"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TimeSinceLastReadMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalMessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalSizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalTimeLagMsByLastRead"},"value":4929},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/UserPartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastReadOld"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MessageLagByCommitted"},"value":30},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionMaxReadQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesAvailAvgMin"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesAvailAvgSec"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesQuota"},"value":1000000000},{"kind":"RATE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadOffsetRewindSum"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadTimeLagMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SizeLagByCommitted"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TimeSinceLastReadMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalMessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalSizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalTimeLagMsByLastRead"},"value":4929},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/UserPartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastReadOld"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/MessageLagByCommitted"},"value":30},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/MessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/PartitionMaxReadQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesAvailAvgMin"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesAvailAvgSec"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadBytesQuota"},"value":1000000000},{"kind":"RATE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadOffsetRewindSum"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/ReadTimeLagMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/SizeLagByCommitted"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/SizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/TimeSinceLastReadMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/TotalMessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/TotalSizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/TotalTimeLagMsByLastRead"},"value":4929},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/UserPartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/WriteTimeLagMsByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"total","important":"total","topic":"total","sensor":"PQ/WriteTimeLagMsByLastReadOld"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MessageLagByCommitted"},"value":30},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionMaxReadQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesAvailAvgMin"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesAvailAvgSec"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesMaxPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadBytesQuota"},"value":1000000000},{"kind":"RATE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadOffsetRewindSum"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/ReadTimeLagMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SizeLagByCommitted"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TimeSinceLastReadMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalMessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalSizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalTimeLagMsByLastRead"},"value":4929},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/UserPartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastReadOld"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/MessageLagByCommitted"},"value":30},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/MessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/PartitionMaxReadQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesAvailAvgMin"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesAvailAvgSec"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesMaxPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesMaxPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesMaxPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesMaxPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadBytesQuota"},"value":1000000000},{"kind":"RATE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadOffsetRewindSum"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/ReadTimeLagMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/SizeLagByCommitted"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/SizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/TimeSinceLastReadMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/TotalMessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/TotalSizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/TotalTimeLagMsByLastRead"},"value":4929},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/UserPartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/WriteTimeLagMsByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"0","topic":"total","sensor":"PQ/WriteTimeLagMsByLastReadOld"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/MessageLagByCommitted"},"value":30},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/MessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/PartitionMaxReadQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesAvailAvgMin"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesAvailAvgSec"},"value":1000000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesMaxPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesPerDay"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesPerHour"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesPerMin"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesPerSec"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadBytesQuota"},"value":1000000000},{"kind":"RATE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadOffsetRewindSum"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/ReadTimeLagMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/SizeLagByCommitted"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/SizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/TimeSinceLastReadMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/TotalMessageLagByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/TotalSizeLagByLastRead"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/TotalTimeLagMsByLastRead"},"value":4929},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/UserPartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/WriteTimeLagMsByLastRead"},"value":29},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","client":"user","important":"total","topic":"total","sensor":"PQ/WriteTimeLagMsByLastReadOld"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/GapsCount"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/GapsMaxCount"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/GapsMaxSize"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/GapsSize"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/MaxPartSize"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionInitTimeMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionLifeTimeMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionMaxWriteQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/PartitionsTotal"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesMaxPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesMaxPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesMaxPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesMaxPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/QuotaBytesPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SourceIdCount"},"value":3},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SourceIdMaxCount"},"value":3},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/SourceIdMinLifetimeMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/TotalPartSize"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesAvailAvgMin"},"value":49999998},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesAvailAvgSec"},"value":50000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesMaxPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesMaxPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesMaxPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesMaxPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteBytesQuota"},"value":50000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"rt3.dc1--asdfgs--topic","sensor":"PQ/WriteTimeLagMsByLastWrite"},"value":32},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/GapsCount"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/GapsMaxCount"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/GapsMaxSize"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/GapsSize"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/MaxPartSize"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/PartitionInitTimeMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/PartitionLifeTimeMs"},"value":5000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/PartitionMaxWriteQuotaUsage"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/PartitionsAnswered"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/PartitionsTotal"},"value":2},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesMaxPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesMaxPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesMaxPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesMaxPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/QuotaBytesPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/SourceIdCount"},"value":3},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/SourceIdMaxCount"},"value":3},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/SourceIdMinLifetimeMs"},"value":0},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/TotalPartSize"},"value":744},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesAvailAvgMin"},"value":49999998},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesAvailAvgSec"},"value":50000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesMaxPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesMaxPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesMaxPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesMaxPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesPerDay"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesPerHour"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesPerMin"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesPerSec"},"value":540},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteBytesQuota"},"value":50000000},{"kind":"GAUGE","labels":{"user_counters":"PersQueue","topic":"total","sensor":"PQ/WriteTimeLagMsByLastWrite"},"value":32}]} +{ + "sensors": [ + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MessageLagByCommitted" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionMaxReadQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesAvailAvgMin" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesAvailAvgSec" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesQuota" + }, + "value": 1000000000 + }, + { + "kind": "RATE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadOffsetRewindSum" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadTimeLagMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SizeLagByCommitted" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TimeSinceLastReadMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalMessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalSizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalTimeLagMsByLastRead" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/UserPartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastRead" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastReadOld" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MessageLagByCommitted" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionMaxReadQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesAvailAvgMin" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesAvailAvgSec" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesQuota" + }, + "value": 1000000000 + }, + { + "kind": "RATE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadOffsetRewindSum" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadTimeLagMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SizeLagByCommitted" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TimeSinceLastReadMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalMessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalSizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalTimeLagMsByLastRead" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/UserPartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastRead" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastReadOld" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/MessageLagByCommitted" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/MessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/PartitionMaxReadQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesAvailAvgMin" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesAvailAvgSec" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesQuota" + }, + "value": 1000000000 + }, + { + "kind": "RATE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadOffsetRewindSum" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadTimeLagMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/SizeLagByCommitted" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/SizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/TimeSinceLastReadMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/TotalMessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/TotalSizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/TotalTimeLagMsByLastRead" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/UserPartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastRead" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "total", + "important": "total", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastReadOld" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MessageLagByCommitted" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionMaxReadQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesAvailAvgMin" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesAvailAvgSec" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesMaxPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadBytesQuota" + }, + "value": 1000000000 + }, + { + "kind": "RATE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadOffsetRewindSum" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReadTimeLagMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SizeLagByCommitted" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TimeSinceLastReadMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalMessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalSizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalTimeLagMsByLastRead" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/UserPartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastRead" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastReadOld" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/MessageLagByCommitted" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/MessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/PartitionMaxReadQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesAvailAvgMin" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesAvailAvgSec" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadBytesQuota" + }, + "value": 1000000000 + }, + { + "kind": "RATE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadOffsetRewindSum" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/ReadTimeLagMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/SizeLagByCommitted" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/SizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/TimeSinceLastReadMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/TotalMessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/TotalSizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/TotalTimeLagMsByLastRead" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/UserPartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastRead" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "0", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastReadOld" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/MessageLagByCommitted" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/MessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/PartitionMaxReadQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesAvailAvgMin" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesAvailAvgSec" + }, + "value": 1000000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesMaxPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerDay" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerHour" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerMin" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesPerSec" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadBytesQuota" + }, + "value": 1000000000 + }, + { + "kind": "RATE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadOffsetRewindSum" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/ReadTimeLagMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/SizeLagByCommitted" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/SizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/TimeSinceLastReadMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/TotalMessageLagByLastRead" + }, + "value": 29 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/TotalSizeLagByLastRead" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/TotalTimeLagMsByLastRead" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/UserPartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastRead" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "client": "user", + "important": "total", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastReadOld" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/GapsCount" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/GapsMaxCount" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/GapsMaxSize" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/GapsSize" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/MaxPartSize" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionInitTimeMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionLifeTimeMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionMaxWriteQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/PartitionsTotal" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesMaxPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesMaxPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesMaxPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesMaxPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/QuotaBytesPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReserveLimitBytes" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/ReserveUsedBytes" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SourceIdCount" + }, + "value": 3 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SourceIdMaxCount" + }, + "value": 3 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/SourceIdMinLifetimeMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/TotalPartSize" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesAvailAvgMin" + }, + "value": 49999994 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesAvailAvgSec" + }, + "value": 50000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesMaxPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesMaxPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesMaxPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesMaxPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteBytesQuota" + }, + "value": 50000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "rt3.dc1--asdfgs--topic", + "sensor": "PQ/WriteTimeLagMsByLastWrite" + }, + "value": 30 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/GapsCount" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/GapsMaxCount" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/GapsMaxSize" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/GapsSize" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/MaxPartSize" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/PartitionInitTimeMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/PartitionLifeTimeMs" + }, + "value": 5000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/PartitionMaxWriteQuotaUsage" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/PartitionsAnswered" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/PartitionsTotal" + }, + "value": 2 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesMaxPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesMaxPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesMaxPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesMaxPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/QuotaBytesPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/ReserveLimitBytes" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/ReserveUsedBytes" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/SourceIdCount" + }, + "value": 3 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/SourceIdMaxCount" + }, + "value": 3 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/SourceIdMinLifetimeMs" + }, + "value": 0 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/TotalPartSize" + }, + "value": 747 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesAvailAvgMin" + }, + "value": 49999994 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesAvailAvgSec" + }, + "value": 50000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesMaxPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesMaxPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesMaxPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesMaxPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesPerDay" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesPerHour" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesPerMin" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesPerSec" + }, + "value": 540 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteBytesQuota" + }, + "value": 50000000 + }, + { + "kind": "GAUGE", + "labels": { + "user_counters": "PersQueue", + "topic": "total", + "sensor": "PQ/WriteTimeLagMsByLastWrite" + }, + "value": 30 + } + ] +} diff --git a/ydb/core/persqueue/ut/resources/counters_topics.html b/ydb/core/persqueue/ut/resources/counters_topics.html index fc7e7bc5bd..b91adb4f0c 100644 --- a/ydb/core/persqueue/ut/resources/counters_topics.html +++ b/ydb/core/persqueue/ut/resources/counters_topics.html @@ -24,6 +24,8 @@ host=: name=topic.partition.write.speed_limit_bytes_per_second: 50000000 name=topic.partition.write.throttled_nanoseconds_max: 0 name=topic.producers_count: 3 + name=topic.reserve.limit_bytes: 0 + name=topic.reserve.used_bytes: 0 name=topic.storage_bytes: 747 consumer=client: diff --git a/ydb/core/persqueue/utils.cpp b/ydb/core/persqueue/utils.cpp index 285640ffe6..71c155049e 100644 --- a/ydb/core/persqueue/utils.cpp +++ b/ydb/core/persqueue/utils.cpp @@ -3,6 +3,10 @@ namespace NKikimr::NPQ { ui64 TopicPartitionReserveSize(const NKikimrPQ::TPQTabletConfig& config) { + if (!config.HasMeteringMode()) { + // Only for federative and dedicated installations + return 0; + } if (NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS == config.GetMeteringMode()) { return 0; } @@ -13,6 +17,10 @@ ui64 TopicPartitionReserveSize(const NKikimrPQ::TPQTabletConfig& config) { } ui64 TopicPartitionReserveThroughput(const NKikimrPQ::TPQTabletConfig& config) { + if (!config.HasMeteringMode()) { + // Only for federative and dedicated installations + return 0; + } if (NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS == config.GetMeteringMode()) { return 0; } diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index b074f6e016..78ac72adc7 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -770,7 +770,7 @@ message TFeatureFlags { optional bool EnableChangefeedInitialScan = 77 [default = false]; reserved 78; // EnableKqpScanQuerySourceRead optional bool EnableDynamicNodeAuthorization = 79 [default = false]; - optional bool EnableKqpImmediateEffects = 80 [default = false]; + reserved 80; // EnableKqpImmediateEffect optional bool EnableDataShardGenericReadSets = 81 [default = false]; // enable alter database operation to create subdomain's system tablets // directly in subdomain's hive @@ -779,9 +779,10 @@ message TFeatureFlags { optional bool EnableSmallDiskOptimization = 84 [default = true]; optional bool EnableDataShardVolatileTransactions = 85 [default = false]; optional bool EnableTopicDiskSubDomainQuota = 89 [default = false]; + optional bool EnableChangefeedDynamoDBStreamsFormat = 96 [default = false]; + optional bool EnableGetNodeLabels = 99 [default = false]; } - message THttpProxyConfig { optional bool Enabled = 1; optional uint32 Port = 2; @@ -1247,6 +1248,7 @@ message TTableServiceConfig { optional bool EnableKqpScanQueryStreamIdxLookupJoin = 35 [default = false]; optional bool EnablePredicateExtractForScanQueries = 36 [default = true]; optional bool EnablePredicateExtractForDataQueries = 37 [default = true]; + optional bool EnableKqpImmediateEffects = 38 [default = false]; }; // Config describes immediate controls and allows @@ -1709,6 +1711,10 @@ message TClientCertificateAuthorization { optional TDynamicNodeDefinition DynamicNodeAuthorization = 1; } +message TAwsCompatibilityConfig { + optional string AwsRegion = 1; +} + message TLabel { optional string Name = 1; optional string Value = 2; @@ -1776,6 +1782,7 @@ message TAppConfig { optional NYq.NConfig.TConfig YandexQueryConfig = 50; // TODO: remove after migration to FederatedQueryConfig + optional TAwsCompatibilityConfig AwsCompatibilityConfig = 70; repeated TNamedConfig NamedConfigs = 100; optional string ClusterYamlConfig = 101; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index e1e4337673..5c1d0c039d 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -2,7 +2,8 @@ import "library/cpp/actors/protos/actors.proto"; import "ydb/core/protos/config.proto"; import "ydb/core/protos/console_base.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; -import "ydb/public/api/protos/draft/ydb_console.proto"; +import "ydb/public/api/protos/draft/ydb_dynamic_config.proto"; +import "ydb/public/api/protos/ydb_issue_message.proto"; package NKikimrConsole; option java_package = "ru.yandex.kikimr.proto"; @@ -117,6 +118,7 @@ message TConfigItem { SchemeShardConfigItem = 54; ClientCertificateAuthorizationConfigItem = 55; YamlConfigEnabledItem = 63; + AwsCompatibilityConfigItem = 70; NamedConfigsItem = 100; ClusterYamlConfigItem = 101; @@ -230,63 +232,105 @@ message TConfigureResponse { repeated TAffectedConfig AffectedConfigs = 3; } -message TApplyConfigRequest { - optional Ydb.Console.ApplyConfigRequest Request = 1; +message TSetYamlConfigRequest { + optional Ydb.DynamicConfig.SetConfigRequest Request = 1; optional bytes UserToken = 2; } -message TApplyConfigResponse { - optional Ydb.Console.ApplyConfigResponse Response = 1; +message TSetYamlConfigResponse { +} + +message TReplaceYamlConfigRequest { + optional Ydb.DynamicConfig.ReplaceConfigRequest Request = 1; + optional bytes UserToken = 2; +} + +message TReplaceYamlConfigResponse { +} + +message TDropConfigRequest { + optional Ydb.DynamicConfig.DropConfigRequest Request = 1; + optional bytes UserToken = 2; +} + +message TDropConfigResponse { } message TAddVolatileConfigRequest { - optional Ydb.Console.AddVolatileConfigRequest Request = 1; + optional Ydb.DynamicConfig.AddVolatileConfigRequest Request = 1; optional bytes UserToken = 2; } message TAddVolatileConfigResponse { - optional Ydb.Console.AddVolatileConfigResponse Response = 1; } message TGetAllConfigsRequest { - optional Ydb.Console.GetConfigRequest Request = 1; + optional Ydb.DynamicConfig.GetConfigRequest Request = 1; optional bytes UserToken = 2; } message TGetAllConfigsResponse { - optional Ydb.Console.GetConfigResponse Response = 1; + optional Ydb.DynamicConfig.GetConfigResult Response = 1; +} + +message TGetNodeLabelsRequest { + optional Ydb.DynamicConfig.GetNodeLabelsRequest Request = 1; + optional bytes UserToken = 2; +} + +message TGetNodeLabelsResponse { + optional Ydb.DynamicConfig.GetNodeLabelsResult Response = 1; +} + +message TGetAllMetadataRequest { + optional Ydb.DynamicConfig.GetMetadataRequest Request = 1; + optional bytes UserToken = 2; +} + +message TGetAllMetadataResponse { + optional Ydb.DynamicConfig.GetMetadataResult Response = 1; } message TRemoveVolatileConfigRequest { - optional Ydb.Console.RemoveVolatileConfigRequest Request = 1; + optional Ydb.DynamicConfig.RemoveVolatileConfigRequest Request = 1; optional bytes UserToken = 2; } message TRemoveVolatileConfigResponse { - optional Ydb.Console.RemoveVolatileConfigResponse Response = 1; } message TResolveConfigRequest { - optional Ydb.Console.ResolveConfigRequest Request = 1; + optional Ydb.DynamicConfig.ResolveConfigRequest Request = 1; optional bytes UserToken = 2; } message TResolveConfigResponse { - optional Ydb.Console.ResolveConfigResponse Response = 1; + optional Ydb.DynamicConfig.ResolveConfigResult Response = 1; } message TResolveAllConfigRequest { - optional Ydb.Console.ResolveAllConfigRequest Request = 1; + optional Ydb.DynamicConfig.ResolveAllConfigRequest Request = 1; optional bytes UserToken = 2; } message TResolveAllConfigResponse { - optional Ydb.Console.ResolveAllConfigResponse Response = 1; + optional Ydb.DynamicConfig.ResolveAllConfigResult Response = 1; } message TGetYamlConfigRequest { } +message TUnauthorized { +} + +message TDisabled { +} + +message TGenericError { + repeated Ydb.Issue.IssueMessage Issues = 1; + optional Ydb.StatusIds.StatusCode YdbStatus = 2; +} + message TVolatileYamlConfig { optional uint64 Id = 1; optional string Config = 2; @@ -348,6 +392,7 @@ message TGetNodeConfigRequest { // Filter items by specified kinds (return full config by default). repeated uint32 ItemKinds = 2; optional bool ServeYaml = 3; + optional uint32 YamlApiVersion = 4; } message TGetNodeConfigResponse { @@ -384,6 +429,7 @@ message TConfigSubscriptionRequest { optional bool ServeYaml = 5; optional uint64 YamlVersion = 6; repeated TVolatileYamlConfigVersion VolatileYamlVersion = 7; + optional uint32 YamlApiVersion = 8; } message TConfigSubscriptionResponse { @@ -489,6 +535,8 @@ message TConfigNotificationRequest { // Updated config. optional NKikimrConfig.TAppConfig Config = 3; repeated uint32 ItemKinds = 4; + // Used to show that message are sent from ConfigsDispatcher + optional bool Local = 5; } message TConfigNotificationResponse { @@ -531,6 +579,7 @@ message TToggleConfigValidatorRequest { message TToggleConfigValidatorResponse { optional TStatus Status = 1; } + enum EValidationLevel { // Validation is disabled. VALIDATE_NONE = 0; diff --git a/ydb/core/protos/counters_bs_controller.proto b/ydb/core/protos/counters_bs_controller.proto index af6de7bcf6..542332cc6c 100644 --- a/ydb/core/protos/counters_bs_controller.proto +++ b/ydb/core/protos/counters_bs_controller.proto @@ -193,6 +193,52 @@ enum EPercentileCounters { Ranges { Value: 18000 Name: "18000" } Ranges { Value: 21600 Name: "21600" } }]; + + COUNTER_NUM_REPLICATING_VDISKS_ROT = 4 [(CounterOpts) = { + Name: "NumReplicatingVDisks/ROT" + Integral: true + Ranges { Value: 0 Name: "0" } + Ranges { Value: 600 Name: "600" } + Ranges { Value: 1200 Name: "1200" } + Ranges { Value: 1800 Name: "1800" } + Ranges { Value: 2400 Name: "2400" } + Ranges { Value: 3000 Name: "3000" } + Ranges { Value: 3600 Name: "3600" } + Ranges { Value: 5400 Name: "5400" } + Ranges { Value: 7200 Name: "7200" } + Ranges { Value: 9000 Name: "9000" } + Ranges { Value: 10800 Name: "10800" } + Ranges { Value: 14400 Name: "14400" } + Ranges { Value: 18000 Name: "18000" } + Ranges { Value: 21600 Name: "21600" } + Ranges { Value: 43200 Name: "43200" } + Ranges { Value: 86400 Name: "86400" } + Ranges { Value: 172800 Name: "172800" } + Ranges { Value: 259200 Name: "259200" } + }]; + + COUNTER_NUM_REPLICATING_VDISKS_OTHER = 5 [(CounterOpts) = { + Name: "NumReplicatingVDisks/Other" + Integral: true + Ranges { Value: 0 Name: "0" } + Ranges { Value: 600 Name: "600" } + Ranges { Value: 1200 Name: "1200" } + Ranges { Value: 1800 Name: "1800" } + Ranges { Value: 2400 Name: "2400" } + Ranges { Value: 3000 Name: "3000" } + Ranges { Value: 3600 Name: "3600" } + Ranges { Value: 5400 Name: "5400" } + Ranges { Value: 7200 Name: "7200" } + Ranges { Value: 9000 Name: "9000" } + Ranges { Value: 10800 Name: "10800" } + Ranges { Value: 14400 Name: "14400" } + Ranges { Value: 18000 Name: "18000" } + Ranges { Value: 21600 Name: "21600" } + Ranges { Value: 43200 Name: "43200" } + Ranges { Value: 86400 Name: "86400" } + Ranges { Value: 172800 Name: "172800" } + Ranges { Value: 259200 Name: "259200" } + }]; } enum ETxTypes { diff --git a/ydb/core/protos/counters_pq.proto b/ydb/core/protos/counters_pq.proto index 981e4820bd..466de907f8 100644 --- a/ydb/core/protos/counters_pq.proto +++ b/ydb/core/protos/counters_pq.proto @@ -228,4 +228,7 @@ enum EPartitionLabeledCounters { METRIC_MIN_SID_LIFETIME = 33 [(LabeledCounterOpts) = {Name: "SourceIdMinLifetimeMs" AggrFunc : EAF_MIN SVName: ""}]; METRIC_PARTITIONS_TOTAL = 34 [(LabeledCounterOpts) = {Name: "PartitionsTotal" AggrFunc : EAF_MAX SVName: "topic.partition.total_count"}]; + + METRIC_RESERVE_LIMIT_BYTES = 35 [(LabeledCounterOpts) = {Name: "ReserveLimitBytes" AggrFunc : EAF_SUM SVName: "topic.reserve.limit_bytes"}]; + METRIC_RESERVE_USED_BYTES = 36 [(LabeledCounterOpts) = {Name: "ReserveUsedBytes" AggrFunc : EAF_SUM SVName: "topic.reserve.used_bytes"}]; } diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index 505a8ed6aa..ad9f705ddb 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -185,6 +185,7 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxAlterExternalDataSource = 150 [(CounterOpts) = {Name: "InFlightOps/AlterExternalDataSource"}]; COUNTER_PQ_STATS_QUEUE_SIZE = 151 [(CounterOpts) = {Name: "PQStatsQueueSize"}]; + COUNTER_DISK_SPACE_TOPICS_TOTAL_BYTES = 152 [(CounterOpts) = {Name: "DiskSpaceTopicsTotalBytes"}]; } enum ECumulativeCounters { diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 7e3d329c39..1ab1a3eaeb 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -325,6 +325,22 @@ message TTTLSettings { optional string UseTiering = 3; } +message TTableReplicationConfig { + enum EReplicationMode { + REPLICATION_MODE_NONE = 0; + REPLICATION_MODE_READ_ONLY = 1; + } + + enum EConsistency { + CONSISTENCY_UNKNOWN = 0; + CONSISTENCY_STRONG = 1; + CONSISTENCY_WEAK = 2; + } + + optional EReplicationMode Mode = 1; + optional EConsistency Consistency = 2; +} + message TTableDescription { optional string Name = 1; optional uint64 Id_Deprecated = 2; // LocalPathId, deprecated @@ -360,6 +376,8 @@ message TTableDescription { repeated TCdcStreamDescription CdcStreams = 38; repeated TSequenceDescription Sequences = 39; + + optional TTableReplicationConfig ReplicationConfig = 40; } message TCompressionOptions { @@ -711,6 +729,7 @@ enum ECdcStreamFormat { ECdcStreamFormatInvalid = 0; ECdcStreamFormatProto = 1; ECdcStreamFormatJson = 2; + ECdcStreamFormatDynamoDBStreamsJson = 3; } message TCdcStreamDescription { @@ -721,6 +740,9 @@ message TCdcStreamDescription { optional NKikimrProto.TPathID PathId = 3; optional ECdcStreamState State = 4; optional uint64 SchemaVersion = 5; + repeated TUserAttribute UserAttributes = 8; + // AwsRegion used to mark records in DynamoDB-compatible mode (FormatDynamoDBStreamsJson) + optional string AwsRegion = 9; } message TCreateCdcStream { diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index 4848224b1b..ed2d300245 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -179,6 +179,9 @@ message TSchemeLimits { optional uint64 MaxPQPartitions = 14; optional uint64 MaxTableCdcStreams = 15; + + optional uint64 MaxExports = 16; + optional uint64 MaxImports = 17; } message TEvInitTenantSchemeShard { diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index 2ee2f13a97..7fe189aab0 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -266,6 +266,8 @@ message TKqpReadRangesSourceSettings { optional uint64 LockTxId = 13; optional uint32 LockNodeId = 14; + + optional bool UseFollowers = 17 [default = false]; } message TKqpTaskInfo { diff --git a/ydb/core/security/login_page.cpp b/ydb/core/security/login_page.cpp index bff5d17870..ad5290e3a6 100644 --- a/ydb/core/security/login_page.cpp +++ b/ydb/core/security/login_page.cpp @@ -179,11 +179,25 @@ public: PassAway(); } + TString GetCORS() { + TStringBuilder res; + TString origin = TString(Request.GetHeader("Origin")); + if (origin.empty()) { + origin = "*"; + } + res << "Access-Control-Allow-Origin: " << origin << "\r\n"; + res << "Access-Control-Allow-Credentials: true\r\n"; + res << "Access-Control-Allow-Headers: Content-Type,Authorization,Origin,Accept\r\n"; + res << "Access-Control-Allow-Methods: OPTIONS, GET, POST\r\n"; + return res; + } + void ReplyCookieAndPassAway(const TString& cookie) { TStringStream response; TDuration maxAge = (ToInstant(NLogin::TLoginProvider::GetTokenExpiresAt(cookie)) - TInstant::Now()); response << "HTTP/1.1 200 OK\r\n"; response << "Set-Cookie: ydb_session_id=" << cookie << "; Max-Age=" << maxAge.Seconds() << "\r\n"; + response << GetCORS(); response << "\r\n"; Result.SetValue(MakeHolder<NMon::TEvHttpInfoRes>(response.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); @@ -197,6 +211,7 @@ public: response << "HTTP/1.1 " << status << "\r\n"; response << "Content-Type: application/json\r\n"; response << "Content-Length: " << responseBody.Size() << "\r\n"; + response << GetCORS(); response << "\r\n"; response << responseBody; Result.SetValue(MakeHolder<NMon::TEvHttpInfoRes>(response.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); diff --git a/ydb/core/security/ticket_parser_impl.h b/ydb/core/security/ticket_parser_impl.h index 16d05499a8..6ce4952fb7 100644 --- a/ydb/core/security/ticket_parser_impl.h +++ b/ydb/core/security/ticket_parser_impl.h @@ -1167,10 +1167,19 @@ protected: } template <typename TTokenRecord> + bool CanRefreshAccessServiceTicket(const TTokenRecord& record) { + if (!AccessServiceValidator) { + return false; + } + if (record.TokenType == TDerived::ETokenType::AccessService) { + return (record.Error && record.Error.Retryable) || !record.Signature.AccessKeyId; + } + return record.TokenType == TDerived::ETokenType::Unknown; + } + + template <typename TTokenRecord> bool CanRefreshTicket(const TString& key, TTokenRecord& record) { - if (AccessServiceValidator - && ((record.TokenType == TDerived::ETokenType::AccessService && !record.Signature.AccessKeyId) - || record.TokenType == TDerived::ETokenType::Unknown)) { + if (CanRefreshAccessServiceTicket(record)) { GetDerived()->ResetTokenRecord(record); if (record.Permissions) { RequestAccessServiceAuthorization(key, record); diff --git a/ydb/core/security/ticket_parser_ut.cpp b/ydb/core/security/ticket_parser_ut.cpp index b4590a91c5..b3c5f6be39 100644 --- a/ydb/core/security/ticket_parser_ut.cpp +++ b/ydb/core/security/ticket_parser_ut.cpp @@ -325,6 +325,120 @@ Y_UNIT_TEST_SUITE(TTicketParserTest) { UNIT_ASSERT_VALUES_EQUAL(result->Error.Message, "Service Unavailable"); } + Y_UNIT_TEST(AuthenticationRetryError) { + using namespace Tests; + + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + ui16 servicePort = tp.GetPort(4284); + TString accessServiceEndpoint = "localhost:" + ToString(servicePort); + NKikimrProto::TAuthConfig authConfig; + authConfig.SetUseBlackBox(false); + authConfig.SetUseAccessService(true); + authConfig.SetUseAccessServiceTLS(false); + authConfig.SetAccessServiceEndpoint(accessServiceEndpoint); + authConfig.SetUseStaff(false); + auto settings = TServerSettings(port, authConfig); + settings.SetDomainName("Root"); + settings.CreateTicketParser = NKikimr::CreateTicketParser; + TServer server(settings); + server.EnableGRpc(grpcPort); + server.GetRuntime()->SetLogPriority(NKikimrServices::TICKET_PARSER, NLog::PRI_TRACE); + server.GetRuntime()->SetLogPriority(NKikimrServices::GRPC_CLIENT, NLog::PRI_TRACE); + TClient client(settings); + NClient::TKikimr kikimr(client.GetClientConfig()); + client.InitRootScheme(); + + // Access Server Mock + NKikimr::TAccessServiceMock accessServiceMock; + grpc::ServerBuilder builder; + builder.AddListeningPort(accessServiceEndpoint, grpc::InsecureServerCredentials()).RegisterService(&accessServiceMock); + std::unique_ptr<grpc::Server> accessServer(builder.BuildAndStart()); + + TTestActorRuntime* runtime = server.GetRuntime(); + TActorId sender = runtime->AllocateEdgeActor(); + TAutoPtr<IEventHandle> handle; + + accessServiceMock.ShouldGenerateRetryableError = true; + TEvTicketParser::TEvAuthorizeTicket::TAccessKeySignature signature {.AccessKeyId = "keyId"}; + TEvTicketParser::TEvAuthorizeTicket::TAccessKeySignature retrySignature = signature; + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(std::move(signature), "", {})), 0); + TEvTicketParser::TEvAuthorizeTicketResult* result = runtime->GrabEdgeEvent<TEvTicketParser::TEvAuthorizeTicketResult>(handle); + UNIT_ASSERT(!result->Error.empty()); + UNIT_ASSERT(result->Error.Retryable); + UNIT_ASSERT_VALUES_EQUAL(result->Error.Message, "Service Unavailable"); + + accessServiceMock.ShouldGenerateRetryableError = false; + Sleep(TDuration::Seconds(10)); + + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(std::move(retrySignature), "", {})), 0); + result = runtime->GrabEdgeEvent<TEvTicketParser::TEvAuthorizeTicketResult>(handle); + UNIT_ASSERT(result->Error.empty()); + UNIT_ASSERT(result->Token != nullptr); + UNIT_ASSERT_VALUES_EQUAL(result->Token->GetUserSID(), "user1@as"); + } + + Y_UNIT_TEST(AuthorizationRetryError) { + using namespace Tests; + + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + ui16 servicePort = tp.GetPort(4284); + TString accessServiceEndpoint = "localhost:" + ToString(servicePort); + NKikimrProto::TAuthConfig authConfig; + authConfig.SetUseBlackBox(false); + authConfig.SetUseAccessService(true); + authConfig.SetUseAccessServiceTLS(false); + authConfig.SetAccessServiceEndpoint(accessServiceEndpoint); + authConfig.SetUseStaff(false); + auto settings = TServerSettings(port, authConfig); + settings.SetDomainName("Root"); + settings.CreateTicketParser = NKikimr::CreateTicketParser; + TServer server(settings); + server.EnableGRpc(grpcPort); + server.GetRuntime()->SetLogPriority(NKikimrServices::TICKET_PARSER, NLog::PRI_TRACE); + server.GetRuntime()->SetLogPriority(NKikimrServices::GRPC_CLIENT, NLog::PRI_TRACE); + TClient client(settings); + NClient::TKikimr kikimr(client.GetClientConfig()); + client.InitRootScheme(); + + // Access Server Mock + NKikimr::TAccessServiceMock accessServiceMock; + grpc::ServerBuilder builder; + builder.AddListeningPort(accessServiceEndpoint, grpc::InsecureServerCredentials()).RegisterService(&accessServiceMock); + std::unique_ptr<grpc::Server> accessServer(builder.BuildAndStart()); + + TTestActorRuntime* runtime = server.GetRuntime(); + TActorId sender = runtime->AllocateEdgeActor(); + TAutoPtr<IEventHandle> handle; + + accessServiceMock.ShouldGenerateRetryableError = true; + TEvTicketParser::TEvAuthorizeTicket::TAccessKeySignature signature {.AccessKeyId = "keyId"}; + TEvTicketParser::TEvAuthorizeTicket::TAccessKeySignature retrySignature = signature; + const TVector<TEvTicketParser::TEvAuthorizeTicket::TEntry> entries {{ + TEvTicketParser::TEvAuthorizeTicket::ToPermissions({"something.read"}), + {{"folder_id", "aaaa1234"}, {"database_id", "bbbb4554"}} + }}; + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(std::move(signature), "", entries)), 0); + TEvTicketParser::TEvAuthorizeTicketResult* result = runtime->GrabEdgeEvent<TEvTicketParser::TEvAuthorizeTicketResult>(handle); + UNIT_ASSERT(!result->Error.empty()); + UNIT_ASSERT(result->Error.Retryable); + UNIT_ASSERT_VALUES_EQUAL(result->Error.Message, "Service Unavailable"); + + accessServiceMock.ShouldGenerateRetryableError = false; + Sleep(TDuration::Seconds(10)); + + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(std::move(retrySignature), "", entries)), 0); + result = runtime->GrabEdgeEvent<TEvTicketParser::TEvAuthorizeTicketResult>(handle); + UNIT_ASSERT(result->Error.empty()); + UNIT_ASSERT(result->Token != nullptr); + UNIT_ASSERT_VALUES_EQUAL(result->Token->GetUserSID(), "user1@as"); + UNIT_ASSERT(result->Token->IsExist("something.read-bbbb4554@as")); + UNIT_ASSERT(!result->Token->IsExist("something.write-bbbb4554@as")); + } + Y_UNIT_TEST(AuthenticationUnsupported) { using namespace Tests; diff --git a/ydb/core/tablet/private/aggregated_counters.cpp b/ydb/core/tablet/private/aggregated_counters.cpp index 0416117475..f4d0424a26 100644 --- a/ydb/core/tablet/private/aggregated_counters.cpp +++ b/ydb/core/tablet/private/aggregated_counters.cpp @@ -602,7 +602,8 @@ void TAggregatedLabeledCounters::FromProto( const NKikimrLabeledCounters::TTabletLabeledCounters& labeledCounters) const { for (const auto& counter : labeledCounters.GetLabeledCounter()) { const ui32 nameId{counter.GetNameId()}; - if (strlen(Names[nameId]) != 0) { + + if (nameId < Size() && strlen(Names[nameId]) != 0) { // TODO: ASDFGS if CT_TIMELAG -> ctx.Now() - counters.GetValue const bool derived = counter.GetType() == TLabeledCounterOptions::CT_DERIV; auto namedCounter = group->GetNamedCounter("name", Names[nameId], derived); diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index 111f530b83..a88826428d 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -760,6 +760,8 @@ private: TCounterPtr DatashardSizeBytes; TCounterPtr ResourcesStorageUsedBytes; TCounterPtr ResourcesStorageLimitBytes; + TCounterPtr ResourcesStorageTableUsedBytes; + TCounterPtr ResourcesStorageTopicUsedBytes; TCounterPtr ResourcesStreamUsedShards; TCounterPtr ResourcesStreamLimitShards; //TCounterPtr ResourcesStreamUsedShardsPercents; @@ -786,6 +788,7 @@ private: THistogramPtr ConsumedCpuHistogram; TCounterPtr DiskSpaceTablesTotalBytes; + TCounterPtr DiskSpaceTopicsTotalBytes; TCounterPtr DiskSpaceSoftQuotaBytes; TCounterPtr StreamShardsCount; @@ -827,6 +830,10 @@ private: "resources.storage.used_bytes", false); ResourcesStorageLimitBytes = ydbGroup->GetNamedCounter("name", "resources.storage.limit_bytes", false); + ResourcesStorageTableUsedBytes = ydbGroup->GetNamedCounter("name", + "resources.storage.table.used_bytes", false); + ResourcesStorageTopicUsedBytes = ydbGroup->GetNamedCounter("name", + "resources.storage.topic.used_bytes", false); ResourcesStreamUsedShards = ydbGroup->GetNamedCounter("name", "resources.stream.used_shards", false); @@ -879,6 +886,7 @@ private: auto appGroup = schemeshardGroup->GetSubgroup("category", "app"); DiskSpaceTablesTotalBytes = appGroup->GetCounter("SUM(SchemeShard/DiskSpaceTablesTotalBytes)"); + DiskSpaceTopicsTotalBytes = appGroup->GetCounter("SUM(SchemeShard/DiskSpaceTopicsTotalBytes)"); DiskSpaceSoftQuotaBytes = appGroup->GetCounter("SUM(SchemeShard/DiskSpaceSoftQuotaBytes)"); StreamShardsCount = appGroup->GetCounter("SUM(SchemeShard/StreamShardsCount)"); @@ -886,7 +894,6 @@ private: StreamReservedThroughput = appGroup->GetCounter("SUM(SchemeShard/StreamReservedThroughput)"); StreamReservedStorage = appGroup->GetCounter("SUM(SchemeShard/StreamReservedStorage)"); StreamReservedStorageLimit = appGroup->GetCounter("SUM(SchemeShard/StreamReservedStorageQuota)"); - } } @@ -911,8 +918,15 @@ private: } if (DiskSpaceTablesTotalBytes) { - ResourcesStorageUsedBytes->Set(DiskSpaceTablesTotalBytes->Val()); ResourcesStorageLimitBytes->Set(DiskSpaceSoftQuotaBytes->Val()); + ResourcesStorageTableUsedBytes->Set(DiskSpaceTablesTotalBytes->Val()); + ResourcesStorageTopicUsedBytes->Set(DiskSpaceTopicsTotalBytes->Val()); + + if (AppData()->FeatureFlags.GetEnableTopicDiskSubDomainQuota()) { + ResourcesStorageUsedBytes->Set(ResourcesStorageTableUsedBytes->Val() + ResourcesStorageTopicUsedBytes->Val()); + } else { + ResourcesStorageUsedBytes->Set(ResourcesStorageTableUsedBytes->Val()); + } auto quota = StreamShardsQuota->Val(); ResourcesStreamUsedShards->Set(StreamShardsCount->Val()); diff --git a/ydb/core/tablet/tablet_counters_aggregator_ut.cpp b/ydb/core/tablet/tablet_counters_aggregator_ut.cpp index f94a11f898..8d9318696c 100644 --- a/ydb/core/tablet/tablet_counters_aggregator_ut.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator_ut.cpp @@ -918,6 +918,12 @@ Y_UNIT_TEST_SUITE(TTabletLabeledCountersAggregator) { UNIT_ASSERT_VALUES_EQUAL(pqCounters->GetAggregatedPerTablets().GetLabeledCounter(0).value(), 63); UNIT_ASSERT_VALUES_EQUAL(pqCounters->GetAggregatedPerTablets().GetLabeledCounter(1).value(), 11); + auto additional = pqCounters->MutableAggregatedPerTablets()->AddLabeledCounter(); + additional->SetNameId(1000); + additional->SetValue(13); + additional->SetType(TLabeledCounterOptions::CT_SIMPLE); + additional->SetAggregateFunc(TLabeledCounterOptions::EAF_SUM); + PQCounters.FromProto(counters); } } diff --git a/ydb/core/testlib/CMakeLists.darwin.txt b/ydb/core/testlib/CMakeLists.darwin.txt index ca2ac9ef8d..eabc9495bb 100644 --- a/ydb/core/testlib/CMakeLists.darwin.txt +++ b/ydb/core/testlib/CMakeLists.darwin.txt @@ -108,4 +108,5 @@ target_sources(ydb-core-testlib PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tablet_helpers.cpp ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tenant_runtime.cpp ${CMAKE_SOURCE_DIR}/ydb/core/testlib/test_client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tx_helpers.cpp ) diff --git a/ydb/core/testlib/CMakeLists.linux-aarch64.txt b/ydb/core/testlib/CMakeLists.linux-aarch64.txt index 3c36477a64..a635e10325 100644 --- a/ydb/core/testlib/CMakeLists.linux-aarch64.txt +++ b/ydb/core/testlib/CMakeLists.linux-aarch64.txt @@ -109,4 +109,5 @@ target_sources(ydb-core-testlib PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tablet_helpers.cpp ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tenant_runtime.cpp ${CMAKE_SOURCE_DIR}/ydb/core/testlib/test_client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tx_helpers.cpp ) diff --git a/ydb/core/testlib/CMakeLists.linux.txt b/ydb/core/testlib/CMakeLists.linux.txt index 3c36477a64..a635e10325 100644 --- a/ydb/core/testlib/CMakeLists.linux.txt +++ b/ydb/core/testlib/CMakeLists.linux.txt @@ -109,4 +109,5 @@ target_sources(ydb-core-testlib PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tablet_helpers.cpp ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tenant_runtime.cpp ${CMAKE_SOURCE_DIR}/ydb/core/testlib/test_client.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/testlib/tx_helpers.cpp ) diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index 482b7b97e6..8ae97dc985 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -144,6 +144,7 @@ namespace NActors { nodeAppData->SchemeShardConfig = app0->SchemeShardConfig; nodeAppData->DataShardConfig = app0->DataShardConfig; nodeAppData->MeteringConfig = app0->MeteringConfig; + nodeAppData->AwsCompatibilityConfig = app0->AwsCompatibilityConfig; nodeAppData->EnableMvccSnapshotWithLegacyDomainRoot = app0->EnableMvccSnapshotWithLegacyDomainRoot; nodeAppData->IoContextFactory = app0->IoContextFactory; if (KeyConfigGenerator) { diff --git a/ydb/core/testlib/basics/appdata.cpp b/ydb/core/testlib/basics/appdata.cpp index 1f8c0a2903..89545d0165 100644 --- a/ydb/core/testlib/basics/appdata.cpp +++ b/ydb/core/testlib/basics/appdata.cpp @@ -56,6 +56,7 @@ namespace NKikimr { app->DataShardConfig = DataShardConfig; app->SchemeShardConfig = SchemeShardConfig; app->MeteringConfig = MeteringConfig; + app->AwsCompatibilityConfig = AwsCompatibilityConfig; app->FeatureFlags = FeatureFlags; // This is a special setting active in test runtime only @@ -192,4 +193,9 @@ namespace NKikimr { { FeatureFlags.SetEnableDbCounters(value); } + + void TAppPrepare::SetAwsRegion(const TString& value) + { + AwsCompatibilityConfig.SetAwsRegion(value); + } } diff --git a/ydb/core/testlib/basics/appdata.h b/ydb/core/testlib/basics/appdata.h index cdf1cf9bd0..1780966cc2 100644 --- a/ydb/core/testlib/basics/appdata.h +++ b/ydb/core/testlib/basics/appdata.h @@ -80,6 +80,7 @@ namespace NKikimr { void SetEnableProtoSourceIdInfo(std::optional<bool> value); void SetEnablePqBilling(std::optional<bool> value); void SetEnableDbCounters(bool value); + void SetAwsRegion(const TString& value); TIntrusivePtr<TChannelProfiles> Channels; NKikimrBlobStorage::TNodeWardenServiceSet BSConf; @@ -93,6 +94,7 @@ namespace NKikimr { NKikimrConfig::TSchemeShardConfig SchemeShardConfig; NKikimrConfig::TMeteringConfig MeteringConfig; NKikimrPQ::TPQConfig PQConfig; + NKikimrConfig::TAwsCompatibilityConfig AwsCompatibilityConfig; private: TAutoPtr<TMine> Mine; diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index 9b8a279b1c..f1869d194f 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -38,11 +38,11 @@ public: FEATURE_FLAG_SETTER(EnableArrowFormatAtDatashard) FEATURE_FLAG_SETTER(EnableGrpcAudit) FEATURE_FLAG_SETTER(EnableChangefeedInitialScan) - FEATURE_FLAG_SETTER(EnableKqpImmediateEffects) FEATURE_FLAG_SETTER(EnableDataShardGenericReadSets) FEATURE_FLAG_SETTER(EnableAlterDatabaseCreateHiveFirst) FEATURE_FLAG_SETTER(EnableDataShardVolatileTransactions) FEATURE_FLAG_SETTER(EnableTopicDiskSubDomainQuota) + FEATURE_FLAG_SETTER(EnableChangefeedDynamoDBStreamsFormat) #undef FEATURE_FLAG_SETTER }; diff --git a/ydb/core/testlib/basics/helpers.h b/ydb/core/testlib/basics/helpers.h index 9658b83714..96c9197e61 100644 --- a/ydb/core/testlib/basics/helpers.h +++ b/ydb/core/testlib/basics/helpers.h @@ -44,7 +44,7 @@ namespace NFake { ui32 nrings, ui32 ringSize, ui64 stateStorageGroup); void SetupBSNodeWarden(TTestActorRuntime& runtime, ui32 nodeIndex, TIntrusivePtr<TNodeWardenConfig> nodeWardenConfig); void SetupTabletResolver(TTestActorRuntime& runtime, ui32 nodeIndex); - void SetupTabletPipePeNodeCaches(TTestActorRuntime& runtime, ui32 nodeIndex); + void SetupTabletPipePeNodeCaches(TTestActorRuntime& runtime, ui32 nodeIndex, bool forceFollowers = false); void SetupResourceBroker(TTestActorRuntime& runtime, ui32 nodeIndex); void SetupSharedPageCache(TTestActorRuntime& runtime, ui32 nodeIndex, NFake::TCaches caches); void SetupNodeWhiteboard(TTestActorRuntime& runtime, ui32 nodeIndex); @@ -57,7 +57,7 @@ namespace NFake { // StateStorage, NodeWarden, TabletResolver, ResourceBroker, SharedPageCache void SetupBasicServices(TTestActorRuntime &runtime, TAppPrepare &app, bool mockDisk = false, - NFake::INode *factory = nullptr, NFake::TStorage storage = {}, NFake::TCaches caches = {}); + NFake::INode *factory = nullptr, NFake::TStorage storage = {}, NFake::TCaches caches = {}, bool forceFollowers = false); /// class TStrandedPDiskServiceFactory : public IPDiskServiceFactory { diff --git a/ydb/core/testlib/basics/services.cpp b/ydb/core/testlib/basics/services.cpp index 9ffba79e19..7cbff912a7 100644 --- a/ydb/core/testlib/basics/services.cpp +++ b/ydb/core/testlib/basics/services.cpp @@ -66,7 +66,7 @@ namespace NPDisk { TActorSetupCmd(tabletResolver, TMailboxType::Revolving, 0), nodeIndex); } - void SetupTabletPipePeNodeCaches(TTestActorRuntime& runtime, ui32 nodeIndex) + void SetupTabletPipePeNodeCaches(TTestActorRuntime& runtime, ui32 nodeIndex, bool forceFollowers) { TIntrusivePtr<TPipePeNodeCacheConfig> leaderPipeConfig = new TPipePeNodeCacheConfig(); leaderPipeConfig->PipeRefreshTime = TDuration::Zero(); @@ -76,6 +76,7 @@ namespace NPDisk { followerPipeConfig->PipeRefreshTime = TDuration::Seconds(30); followerPipeConfig->PipeConfig.AllowFollower = true; followerPipeConfig->PipeConfig.RetryPolicy = {.RetryLimitCount = 3}; + followerPipeConfig->PipeConfig.ForceFollower = forceFollowers; runtime.AddLocalService(MakePipePeNodeCacheID(false), TActorSetupCmd(CreatePipePeNodeCache(leaderPipeConfig), TMailboxType::Revolving, 0), nodeIndex); @@ -310,7 +311,7 @@ namespace NPDisk { } void SetupBasicServices(TTestActorRuntime& runtime, TAppPrepare& app, bool mock, - NFake::INode* factory, NFake::TStorage storage, NFake::TCaches caches) + NFake::INode* factory, NFake::TStorage storage, NFake::TCaches caches, bool forceFollowers) { runtime.SetDispatchTimeout(storage.UseDisk ? DISK_DISPATCH_TIMEOUT : DEFAULT_DISPATCH_TIMEOUT); @@ -336,7 +337,7 @@ namespace NPDisk { SetupBSNodeWarden(runtime, nodeIndex, disk.MakeWardenConf(*app.Domains, keyConfig)); SetupTabletResolver(runtime, nodeIndex); - SetupTabletPipePeNodeCaches(runtime, nodeIndex); + SetupTabletPipePeNodeCaches(runtime, nodeIndex, forceFollowers); SetupResourceBroker(runtime, nodeIndex); SetupSharedPageCache(runtime, nodeIndex, caches); SetupBlobCache(runtime, nodeIndex); diff --git a/ydb/core/testlib/cs_helper.cpp b/ydb/core/testlib/cs_helper.cpp index 42c2969113..aa53eb6d21 100644 --- a/ydb/core/testlib/cs_helper.cpp +++ b/ydb/core/testlib/cs_helper.cpp @@ -320,7 +320,8 @@ std::shared_ptr<arrow::Schema> TTableWithNullsHelper::GetArrowSchema() { std::vector<std::shared_ptr<arrow::Field>>{ arrow::field("id", arrow::int32()), arrow::field("resource_id", arrow::utf8()), - arrow::field("level", arrow::int32()) + arrow::field("level", arrow::int32()), + arrow::field("binary_str", arrow::binary()) }); } @@ -335,28 +336,33 @@ std::shared_ptr<arrow::RecordBatch> TTableWithNullsHelper::TestArrowBatch(ui64, arrow::Int32Builder b1; arrow::StringBuilder b2; arrow::Int32Builder b3; + arrow::StringBuilder b4; for (size_t i = 1; i <= rowCount / 2; ++i) { Y_VERIFY(b1.Append(i).ok()); Y_VERIFY(b2.AppendNull().ok()); Y_VERIFY(b3.Append(i).ok()); + Y_VERIFY(b4.AppendNull().ok()); } for (size_t i = rowCount / 2 + 1; i <= rowCount; ++i) { Y_VERIFY(b1.Append(i).ok()); Y_VERIFY(b2.Append(std::to_string(i)).ok()); Y_VERIFY(b3.AppendNull().ok()); + Y_VERIFY(b4.Append(std::to_string(i)).ok()); } std::shared_ptr<arrow::Int32Array> a1; std::shared_ptr<arrow::StringArray> a2; std::shared_ptr<arrow::Int32Array> a3; + std::shared_ptr<arrow::StringArray> a4; Y_VERIFY(b1.Finish(&a1).ok()); Y_VERIFY(b2.Finish(&a2).ok()); Y_VERIFY(b3.Finish(&a3).ok()); + Y_VERIFY(b4.Finish(&a4).ok()); - return arrow::RecordBatch::Make(schema, rowCount, { a1, a2, a3 }); + return arrow::RecordBatch::Make(schema, rowCount, { a1, a2, a3, a4 }); } } diff --git a/ydb/core/testlib/cs_helper.h b/ydb/core/testlib/cs_helper.h index 1e7738c9e1..028e21f3fb 100644 --- a/ydb/core/testlib/cs_helper.h +++ b/ydb/core/testlib/cs_helper.h @@ -179,6 +179,7 @@ public: Columns { Name: "id" Type: "Int32" NotNull: true } Columns { Name: "resource_id" Type: "Utf8" } Columns { Name: "level" Type: "Int32" } + Columns { Name: "binary_str" Type: "String" } KeyColumnNames: "id" )"; diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index 41d4f3a391..bb0a57971d 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -618,13 +618,13 @@ namespace NKikimr { } void SetupTabletServices(TTestActorRuntime &runtime, TAppPrepare *app, bool mockDisk, NFake::TStorage storage, - NFake::TCaches caches) { + NFake::TCaches caches, bool forceFollowers) { TAutoPtr<TAppPrepare> dummy; if (app == nullptr) { dummy = app = new TAppPrepare; } TUltimateNodes nodes(runtime, app); - SetupBasicServices(runtime, *app, mockDisk, &nodes, storage, caches); + SetupBasicServices(runtime, *app, mockDisk, &nodes, storage, caches, forceFollowers); } TDomainsInfo::TDomain::TStoragePoolKinds DefaultPoolKinds(ui32 count) { diff --git a/ydb/core/testlib/tablet_helpers.h b/ydb/core/testlib/tablet_helpers.h index 81a7c614aa..9e3616148e 100644 --- a/ydb/core/testlib/tablet_helpers.h +++ b/ydb/core/testlib/tablet_helpers.h @@ -26,7 +26,7 @@ namespace NKikimr { void RebootTablet(TTestActorRuntime& runtime, ui64 tabletId, const TActorId& sender, ui32 nodeIndex = 0, bool sysTablet = false); void GracefulRestartTablet(TTestActorRuntime& runtime, ui64 tabletId, const TActorId& sender, ui32 nodeIndex = 0); void SetupTabletServices(TTestActorRuntime& runtime, TAppPrepare* app = nullptr, bool mockDisk = false, - NFake::TStorage storage = {}, NFake::TCaches caches = {}); + NFake::TStorage storage = {}, NFake::TCaches caches = {}, bool forceFollowers = false); const TString DEFAULT_STORAGE_POOL = "Storage Pool with id: 1"; diff --git a/ydb/core/testlib/tenant_runtime.cpp b/ydb/core/testlib/tenant_runtime.cpp index 90d1abfef5..1c7f1b68f1 100644 --- a/ydb/core/testlib/tenant_runtime.cpp +++ b/ydb/core/testlib/tenant_runtime.cpp @@ -1004,7 +1004,12 @@ void TTenantTestRuntime::Setup(bool createTenantPools) // Create other local services for (size_t i = 0; i < Config.Nodes.size(); ++i) { if (Config.CreateConfigsDispatcher) { - auto aid = Register(CreateConfigsDispatcher(Extension, {})); + TMap<TString, TString> labels; + for (const auto &label : Extension.GetLabels()) { + labels[label.GetName()] = label.GetValue(); + } + labels.emplace("node_id", ToString(i)); + auto aid = Register(CreateConfigsDispatcher(Extension, labels)); EnableScheduleForActor(aid, true); RegisterService(MakeConfigsDispatcherID(GetNodeId(0)), aid, 0); } diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 3550325b48..a381228d44 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -171,6 +171,7 @@ namespace Tests { app.SetKeepSnapshotTimeout(Settings->KeepSnapshotTimeout); app.SetChangesQueueItemsLimit(Settings->ChangesQueueItemsLimit); app.SetChangesQueueBytesLimit(Settings->ChangesQueueBytesLimit); + app.SetAwsRegion(Settings->AwsRegion); app.CompactionConfig = Settings->CompactionConfig; app.FeatureFlags = Settings->FeatureFlags; @@ -233,7 +234,7 @@ namespace Tests { }); const bool mockDisk = (StaticNodes() + DynamicNodes()) == 1 && Settings->EnableMockOnSingleNode; - SetupTabletServices(*Runtime, &app, mockDisk, Settings->CustomDiskParams, Settings->CacheParams); + SetupTabletServices(*Runtime, &app, mockDisk, Settings->CustomDiskParams, Settings->CacheParams, Settings->EnableForceFollowers); // WARNING: must be careful about modifying app data after actor system starts @@ -697,7 +698,12 @@ namespace Tests { TMailboxType::Revolving, 0); Runtime->RegisterService(MakeTenantPoolRootID(), poolId, nodeIdx); if (Settings->EnableConfigsDispatcher) { - auto *dispatcher = NConsole::CreateConfigsDispatcher(Settings->AppConfig, {}); + // We overwrite icb settings here to save behavior when configs dispatcher are enabled + NKikimrConfig::TAppConfig initial = Settings->AppConfig; + if (!initial.HasImmediateControlsConfig()) { + initial.MutableImmediateControlsConfig()->CopyFrom(Settings->Controls); + } + auto *dispatcher = NConsole::CreateConfigsDispatcher(initial, {}); auto aid = Runtime->Register(dispatcher, nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0); Runtime->RegisterService(NConsole::MakeConfigsDispatcherID(Runtime->GetNodeId(nodeIdx)), aid); } diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 21de04466e..86bdc2d22d 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -114,6 +114,7 @@ namespace Tests { TLoggerInitializer LoggerInitializer; TStoragePoolKinds StoragePoolTypes; TVector<NKikimrKqp::TKqpSetting> KqpSettings; + bool EnableForceFollowers = false; bool EnableConsole = true; bool EnableNodeBroker = false; bool EnableConfigsDispatcher = true; @@ -132,6 +133,7 @@ namespace Tests { std::shared_ptr<NKikimr::NPQ::TPersQueueMirrorReaderFactory> PersQueueMirrorReaderFactory = std::make_shared<NKikimr::NPQ::TPersQueueMirrorReaderFactory>(); bool EnableMetering = false; TString MeteringFilePath; + TString AwsRegion; std::function<IActor*(const NKikimrProto::TAuthConfig&)> CreateTicketParser = NKikimr::CreateTicketParser; std::shared_ptr<TGrpcServiceFactory> GrpcServiceFactory; @@ -160,6 +162,7 @@ namespace Tests { TServerSettings& SetAppConfig(const NKikimrConfig::TAppConfig value) { AppConfig = value; return *this; } TServerSettings& SetKeyFor(ui32 nodeId, TString keyValue) { NodeKeys[nodeId] = keyValue; return *this; } TServerSettings& SetEnableKqpSpilling(bool value) { EnableKqpSpilling = value; return *this; } + TServerSettings& SetEnableForceFollowers(bool value) { EnableForceFollowers = value; return *this; } TServerSettings& SetDomainPlanResolution(ui64 resolution) { DomainPlanResolution = resolution; return *this; } TServerSettings& SetFeatureFlags(const NKikimrConfig::TFeatureFlags& value) { FeatureFlags = value; return *this; } TServerSettings& SetCompactionConfig(const NKikimrConfig::TCompactionConfig& value) { CompactionConfig = value; return *this; } @@ -170,6 +173,7 @@ namespace Tests { TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; } TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; } TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; } + TServerSettings& SetAwsRegion(const TString& value) { AwsRegion = value; return *this; } TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory ) { diff --git a/ydb/core/testlib/tx_helpers.cpp b/ydb/core/testlib/tx_helpers.cpp new file mode 100644 index 0000000000..9dd4864318 --- /dev/null +++ b/ydb/core/testlib/tx_helpers.cpp @@ -0,0 +1,52 @@ +#include "tx_helpers.h" + +#include <google/protobuf/text_format.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <ydb/core/base/tablet.h> +#include <ydb/core/testlib/tablet_helpers.h> + +namespace NKikimr { + +NKikimrProto::EReplyStatus LocalSchemeTx(TTestActorRuntime& runtime, ui64 tabletId, const TString& schemeChangesStr, bool dryRun, + NTabletFlatScheme::TSchemeChanges& scheme, TString& err) { + TActorId sender = runtime.AllocateEdgeActor(); + + auto evTx = new TEvTablet::TEvLocalSchemeTx; + evTx->Record.SetDryRun(dryRun); + auto schemeChanges = evTx->Record.MutableSchemeChanges(); + bool parseResult = ::google::protobuf::TextFormat::ParseFromString(schemeChangesStr, schemeChanges); + UNIT_ASSERT_C(parseResult, "protobuf parsing failed"); + + ForwardToTablet(runtime, tabletId, sender, evTx); + + TAutoPtr<IEventHandle> handle; + auto event = runtime.GrabEdgeEvent<TEvTablet::TEvLocalSchemeTxResponse>(handle); + UNIT_ASSERT(event); + + err = event->Record.GetErrorReason(); + scheme.CopyFrom(event->Record.GetFullScheme()); + + // emulate enum behavior from proto3 + return static_cast<NKikimrProto::EReplyStatus>(event->Record.GetStatus()); +} + +ui64 GetExecutorCacheSize(TTestActorRuntime& runtime, ui64 tabletId) { + NTabletFlatScheme::TSchemeChanges scheme; + TString err; + NKikimrProto::EReplyStatus status = LocalSchemeTx(runtime, tabletId, "", true, scheme, err); + UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::EReplyStatus::OK); + //Cdbg << scheme << "\n"; + // looking for "Delta { DeltaType: UpdateExecutorInfo ExecutorCacheSize: 33554432 }" + for (ui32 i = 0; i < scheme.DeltaSize(); ++i) { + const auto& d = scheme.GetDelta(i); + if (d.GetDeltaType() == NTabletFlatScheme::TAlterRecord::UpdateExecutorInfo) { + return d.GetExecutorCacheSize(); + } + } + UNIT_ASSERT_C(false, "UpdateExecutorInfo delta record not found"); + return -1; +} + +} // namespace NKikimr diff --git a/ydb/core/testlib/tx_helpers.h b/ydb/core/testlib/tx_helpers.h new file mode 100644 index 0000000000..f3f4145a4a --- /dev/null +++ b/ydb/core/testlib/tx_helpers.h @@ -0,0 +1,12 @@ +#pragma once + +#include <ydb/core/protos/scheme_log.pb.h> +#include <ydb/core/testlib/actors/test_runtime.h> + +namespace NKikimr { + +NKikimrProto::EReplyStatus LocalSchemeTx(TTestActorRuntime& runtime, ui64 tabletId, const TString& schemeChangesStr, bool dryRun, NTabletFlatScheme::TSchemeChanges& scheme, TString& err); + +ui64 GetExecutorCacheSize(TTestActorRuntime& runtime, ui64 tabletId); + +} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/blob.h b/ydb/core/tx/columnshard/blob.h index d37c2cce0e..f46c542ec8 100644 --- a/ydb/core/tx/columnshard/blob.h +++ b/ydb/core/tx/columnshard/blob.h @@ -357,7 +357,11 @@ struct TEvictedBlob { } bool IsExternal() const { - return ExternBlob.IsValid(); + if (State == EEvictState::EXTERN) { + Y_VERIFY(ExternBlob.IsValid()); + return true; + } + return false; } TString ToString() const { diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 44a82e1826..0739271b2c 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -129,7 +129,7 @@ void TColumnShard::Handle(TEvPrivate::TEvPeriodicWakeup::TPtr& ev, const TActorC return; } - if (LastBackActivation < TInstant::Now() - ActivationPeriod) { + if (LastPeriodicBackActivation < TInstant::Now() - ActivationPeriod) { SendWaitPlanStep(GetOutdatedStep()); } @@ -334,6 +334,14 @@ void TColumnShard::SendPeriodicStats() { NOlap::TSnapshot lastIndexUpdate = PrimaryIndex->LastUpdate(); auto activeIndexStats = indexStats.Active(); // data stats excluding inactive and evicted + if (activeIndexStats.Rows < 0 || activeIndexStats.Bytes < 0) { + LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows + << " Bytes: " << activeIndexStats.Bytes << TabletID()); + + activeIndexStats.Rows = (activeIndexStats.Rows < 0) ? 0 : activeIndexStats.Rows; + activeIndexStats.Bytes = (activeIndexStats.Bytes < 0) ? 0 : activeIndexStats.Bytes; + } + tabletStats->SetRowCount(activeIndexStats.Rows); tabletStats->SetDataSize(activeIndexStats.Bytes + TabletCounters->Simple()[COUNTER_COMMITTED_BYTES].Get()); // TODO: we need row/dataSize counters for evicted data (managed by tablet but stored outside) diff --git a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp index 4624b69949..e82345c302 100644 --- a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp +++ b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp @@ -176,7 +176,7 @@ public: switch (Trigger) { case ETriggerActivities::POST_INSERT: - Self->EnqueueBackgroundActivities(false, true); + Self->EnqueueBackgroundActivities(false, TBackgroundActivity::Indexation()); break; case ETriggerActivities::POST_SCHEMA: Self->EnqueueBackgroundActivities(); diff --git a/ydb/core/tx/columnshard/columnshard__stats_scan.h b/ydb/core/tx/columnshard/columnshard__stats_scan.h index f2ba4a97e1..bcaa1cc409 100644 --- a/ydb/core/tx/columnshard/columnshard__stats_scan.h +++ b/ydb/core/tx/columnshard/columnshard__stats_scan.h @@ -155,39 +155,39 @@ private: ui64 tabletId = ReadMetadata->TabletId; TUInt64 tabletIds[NUM_KINDS] = {tabletId, tabletId, tabletId, tabletId, tabletId}; TUInt64 rows[NUM_KINDS] = { - stats.Inserted.Rows, - stats.Compacted.Rows, - stats.SplitCompacted.Rows, - stats.Inactive.Rows, - stats.Evicted.Rows + (ui64)stats.Inserted.Rows, + (ui64)stats.Compacted.Rows, + (ui64)stats.SplitCompacted.Rows, + (ui64)stats.Inactive.Rows, + (ui64)stats.Evicted.Rows }; TUInt64 bytes[NUM_KINDS] = { - stats.Inserted.Bytes, - stats.Compacted.Bytes, - stats.SplitCompacted.Bytes, - stats.Inactive.Bytes, - stats.Evicted.Bytes + (ui64)stats.Inserted.Bytes, + (ui64)stats.Compacted.Bytes, + (ui64)stats.SplitCompacted.Bytes, + (ui64)stats.Inactive.Bytes, + (ui64)stats.Evicted.Bytes }; TUInt64 rawBytes[NUM_KINDS] = { - stats.Inserted.RawBytes, - stats.Compacted.RawBytes, - stats.SplitCompacted.RawBytes, - stats.Inactive.RawBytes, - stats.Evicted.RawBytes + (ui64)stats.Inserted.RawBytes, + (ui64)stats.Compacted.RawBytes, + (ui64)stats.SplitCompacted.RawBytes, + (ui64)stats.Inactive.RawBytes, + (ui64)stats.Evicted.RawBytes }; TUInt64 portions[NUM_KINDS] = { - stats.Inserted.Portions, - stats.Compacted.Portions, - stats.SplitCompacted.Portions, - stats.Inactive.Portions, - stats.Evicted.Portions + (ui64)stats.Inserted.Portions, + (ui64)stats.Compacted.Portions, + (ui64)stats.SplitCompacted.Portions, + (ui64)stats.Inactive.Portions, + (ui64)stats.Evicted.Portions }; TUInt64 blobs[NUM_KINDS] = { - stats.Inserted.Blobs, - stats.Compacted.Blobs, - stats.SplitCompacted.Blobs, - stats.Inactive.Blobs, - stats.Evicted.Blobs + (ui64)stats.Inserted.Blobs, + (ui64)stats.Compacted.Blobs, + (ui64)stats.SplitCompacted.Blobs, + (ui64)stats.Inactive.Blobs, + (ui64)stats.Evicted.Blobs }; if (Reverse) { diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index a259b385b8..b354a4e1cc 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -160,7 +160,8 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex } else { errCode = NKikimrTxColumnShard::EResultStatus::STORAGE_ERROR; } - --WritesInFly; // write failed + --WritesInFlight; // write failed + WritesSizeInFlight -= ev->Get()->ResourceUsage.SourceMemorySize; } auto result = std::make_unique<TEvColumnShard::TEvWriteResult>( @@ -171,7 +172,8 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex LOG_S_DEBUG("Write (record) " << data.size() << " bytes into pathId " << tableId << (writeId? (" writeId " + ToString(writeId)).c_str() : "") << " at tablet " << TabletID()); - --WritesInFly; // write successed + --WritesInFlight; // write successed + WritesSizeInFlight -= ev->Get()->ResourceUsage.SourceMemorySize; Y_VERIFY(putStatus == NKikimrProto::OK); Execute(new TTxWrite(this, ev), ctx); } else if (isOutOfSpace || InsertTable->IsOverloaded(tableId) || ShardOverloaded()) { @@ -190,6 +192,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex LOG_S_INFO("Write (overload) " << data.size() << " bytes into pathId " << tableId << (ShardOverloaded()? " [shard]" : "") << (tableOverload? " [table]" : "") + << " inflight " << WritesInFlight << " (" << WritesSizeInFlight << " bytes)" << " at tablet " << TabletID()); } @@ -214,18 +217,22 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex } } + ev->Get()->MaxSmallBlobSize = Settings.MaxSmallBlobSize; + ev->Get()->ResourceUsage.SourceMemorySize = data.size(); + + ++WritesInFlight; // write started + WritesSizeInFlight += ev->Get()->ResourceUsage.SourceMemorySize; + LOG_S_DEBUG("Write (blob) " << data.size() << " bytes into pathId " << tableId << (writeId? (" writeId " + ToString(writeId)).c_str() : "") + << " inflight " << WritesInFlight << " (" << WritesSizeInFlight << " bytes)" << " at tablet " << TabletID()); - ev->Get()->MaxSmallBlobSize = Settings.MaxSmallBlobSize; - - ++WritesInFly; // write started ctx.Register(CreateWriteActor(TabletID(), PrimaryIndex->GetIndexInfo(), ctx.SelfID, BlobManager->StartBlobBatch(), Settings.BlobWriteGrouppingEnabled, ev->Release())); } - SetCounter(COUNTER_WRITES_IN_FLY, WritesInFly); + SetCounter(COUNTER_WRITES_IN_FLY, WritesInFlight); } } diff --git a/ydb/core/tx/columnshard/columnshard__write_index.cpp b/ydb/core/tx/columnshard/columnshard__write_index.cpp index 0ac47594c9..61149c87fa 100644 --- a/ydb/core/tx/columnshard/columnshard__write_index.cpp +++ b/ydb/core/tx/columnshard/columnshard__write_index.cpp @@ -36,6 +36,7 @@ private: THashMap<TString, TPathIdBlobs> ExportTierBlobs; THashSet<NOlap::TEvictedBlob> BlobsToForget; ui64 ExportNo = 0; + TBackgroundActivity TriggerActivity = TBackgroundActivity::All(); }; @@ -274,12 +275,14 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) Self->IncCounter(COUNTER_COMPACTION_TIME, Ev->Get()->Duration.MilliSeconds()); } else if (changes->IsCleanup()) { Self->ActiveCleanup = false; + TriggerActivity = changes->NeedRepeat ? TBackgroundActivity::Cleanup() : TBackgroundActivity::None(); Self->BlobManager->GetCleanupBlobs(BlobsToForget); Self->IncCounter(ok ? COUNTER_CLEANUP_SUCCESS : COUNTER_CLEANUP_FAIL); } else if (changes->IsTtl()) { Self->ActiveTtl = false; + //TriggerActivity = changes->NeedRepeat ? TBackgroundActivity::Ttl() : TBackgroundActivity::None(); // Do not start new TTL till we evict current PortionsToEvict. We could evict them twice otherwise Y_VERIFY(!Self->ActiveEvictions, "Unexpected active evictions count at tablet %lu", Self->TabletID()); @@ -301,7 +304,7 @@ void TTxWriteIndex::Complete(const TActorContext& ctx) { if (Ev->Get()->PutStatus == NKikimrProto::TRYLATER) { ctx.Schedule(Self->FailActivationDelay, new TEvPrivate::TEvPeriodicWakeup(true)); } else { - Self->EnqueueBackgroundActivities(); + Self->EnqueueBackgroundActivities(false, TriggerActivity); } for (auto& [tierName, pathBlobs] : ExportTierBlobs) { diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 5429040f46..b9128271bf 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -681,15 +681,18 @@ void TColumnShard::ScheduleNextGC(const TActorContext& ctx, bool cleanupOnly) { } } -void TColumnShard::EnqueueBackgroundActivities(bool periodic, bool insertOnly) { - if (periodic && LastBackActivation > TInstant::Now() - ActivationPeriod) { - return; +void TColumnShard::EnqueueBackgroundActivities(bool periodic, TBackgroundActivity activity) { + if (periodic) { + if (LastPeriodicBackActivation > TInstant::Now() - ActivationPeriod) { + return; + } + LastPeriodicBackActivation = TInstant::Now(); } const TActorContext& ctx = TActivationContext::ActorContextFor(SelfId()); SendPeriodicStats(); - if (insertOnly) { + if (activity.IndexationOnly()) { if (auto event = SetupIndexation()) { ctx.Send(IndexingActor, event.release()); } @@ -699,44 +702,56 @@ void TColumnShard::EnqueueBackgroundActivities(bool periodic, bool insertOnly) { // Preventing conflicts between indexing and compaction leads to election between them. // Indexing vs compaction probability depends on index and insert table overload status. // Prefer compaction: 25% by default; 50% if IndexOverloaded(); 6.25% if InsertTableOverloaded(). - ui32 mask = IndexOverloaded() ? 0x1 : 0x3; - if (InsertTableOverloaded()) { - mask = 0x0F; - } - bool preferIndexing = (++BackgroundActivation) & mask; + if (activity.HasIndexation() && activity.HasCompaction()) { + ui32 mask = IndexOverloaded() ? 0x1 : 0x3; + if (InsertTableOverloaded()) { + mask = 0x0F; + } + bool preferIndexing = (++BackgroundActivation) & mask; - if (preferIndexing) { + if (preferIndexing) { + if (auto evIdx = SetupIndexation()) { + ctx.Send(IndexingActor, evIdx.release()); + } else if (auto event = SetupCompaction()) { + ctx.Send(CompactionActor, event.release()); + } + } else { + if (auto event = SetupCompaction()) { + ctx.Send(CompactionActor, event.release()); + } else if (auto evIdx = SetupIndexation()) { + ctx.Send(IndexingActor, evIdx.release()); + } + } + } else if (activity.HasIndexation()) { if (auto evIdx = SetupIndexation()) { ctx.Send(IndexingActor, evIdx.release()); - } else if (auto event = SetupCompaction()) { - ctx.Send(CompactionActor, event.release()); } - } else { + } else if (activity.HasCompaction()) { if (auto event = SetupCompaction()) { ctx.Send(CompactionActor, event.release()); - } else if (auto evIdx = SetupIndexation()) { - ctx.Send(IndexingActor, evIdx.release()); } } - if (auto event = SetupCleanup()) { - ctx.Send(SelfId(), event.release()); - } else { - // Small cleanup (no index changes) - THashSet<NOlap::TEvictedBlob> blobsToForget; - BlobManager->GetCleanupBlobs(blobsToForget); - ForgetBlobs(ctx, blobsToForget); - } - - if (auto event = SetupTtl()) { - if (event->NeedDataReadWrite()) { - ctx.Send(EvictionActor, event.release()); + if (activity.HasCleanup()) { + if (auto event = SetupCleanup()) { + ctx.Send(SelfId(), event.release()); } else { - ctx.Send(SelfId(), event->TxEvent.release()); + // Small cleanup (no index changes) + THashSet<NOlap::TEvictedBlob> blobsToForget; + BlobManager->GetCleanupBlobs(blobsToForget); + ForgetBlobs(ctx, blobsToForget); } } - LastBackActivation = TInstant::Now(); + if (activity.HasTtl()) { + if (auto event = SetupTtl()) { + if (event->NeedDataReadWrite()) { + ctx.Send(EvictionActor, event.release()); + } else { + ctx.Send(SelfId(), event->TxEvent.release()); + } + } + } } std::unique_ptr<TEvPrivate::TEvIndexing> TColumnShard::SetupIndexation() { @@ -843,16 +858,12 @@ std::unique_ptr<TEvPrivate::TEvCompaction> TColumnShard::SetupCompaction() { } PrimaryIndex->UpdateCompactionLimits(CompactionLimits.Get()); - auto compactionInfo = PrimaryIndex->Compact(); + auto compactionInfo = PrimaryIndex->Compact(LastCompactedGranule); if (!compactionInfo || compactionInfo->Empty()) { LOG_S_DEBUG("Compaction not started: no portions to compact at tablet " << TabletID()); return {}; } - // TODO: Compact granules in parallel - - // Rotate compaction granules: do not choose the same granule all the time. - LastCompactedGranule = compactionInfo->ChooseOneGranule(LastCompactedGranule); Y_VERIFY(compactionInfo->Good()); LOG_S_DEBUG("Prepare " << *compactionInfo << " at tablet " << TabletID()); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 48642be2c6..9d015332bd 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -48,16 +48,15 @@ struct TSettings { TControlWrapper CacheDataAfterIndexing; TControlWrapper CacheDataAfterCompaction; TControlWrapper MaxSmallBlobSize; - TControlWrapper OverloadTxInFly; - TControlWrapper OverloadWritesInFly; + static constexpr ui64 OverloadTxInFlight = 1000; + static constexpr ui64 OverloadWritesInFlight = 1000; + static constexpr ui64 OverloadWritesSizeInFlight = 128 * 1024 * 1024; TSettings() : BlobWriteGrouppingEnabled(1, 0, 1) , CacheDataAfterIndexing(1, 0, 1) , CacheDataAfterCompaction(1, 0, 1) , MaxSmallBlobSize(0, 0, 8000000) - , OverloadTxInFly(1000, 0, 10000) - , OverloadWritesInFly(1000, 0, 10000) {} void RegisterControls(TControlBoard& icb) { @@ -65,11 +64,44 @@ struct TSettings { icb.RegisterSharedControl(CacheDataAfterIndexing, "ColumnShardControls.CacheDataAfterIndexing"); icb.RegisterSharedControl(CacheDataAfterCompaction, "ColumnShardControls.CacheDataAfterCompaction"); icb.RegisterSharedControl(MaxSmallBlobSize, "ColumnShardControls.MaxSmallBlobSize"); - icb.RegisterSharedControl(OverloadTxInFly, "ColumnShardControls.OverloadTxInFly"); - icb.RegisterSharedControl(OverloadWritesInFly, "ColumnShardControls.OverloadWritesInFly"); } }; +class TBackgroundActivity { +public: + enum EBackActivity : ui32 { + NONE = 0x00, + INDEX = 0x01, + COMPACT = 0x02, + CLEAN = 0x04, + TTL = 0x08, + ALL = 0xffff + }; + + static TBackgroundActivity Indexation() { return TBackgroundActivity(INDEX); } + static TBackgroundActivity Compaction() { return TBackgroundActivity(COMPACT); } + static TBackgroundActivity Cleanup() { return TBackgroundActivity(CLEAN); } + static TBackgroundActivity Ttl() { return TBackgroundActivity(TTL); } + static TBackgroundActivity All() { return TBackgroundActivity(ALL); } + static TBackgroundActivity None() { return TBackgroundActivity(NONE); } + + TBackgroundActivity() = default; + + bool HasIndexation() const { return Activity & INDEX; } + bool HasCompaction() const { return Activity & COMPACT; } + bool HasCleanup() const { return Activity & CLEAN; } + bool HasTtl() const { return Activity & TTL; } + bool HasAll() const { return Activity == ALL; } + bool IndexationOnly() const { return Activity == INDEX; } + +private: + EBackActivity Activity = NONE; + + TBackgroundActivity(EBackActivity activity) + : Activity(activity) + {} +}; + using ITransaction = NTabletFlatExecutor::ITransaction; template <typename T> @@ -336,7 +368,8 @@ private: ui64 LastPlannedTxId = 0; ui64 LastCompactedGranule = 0; ui64 LastExportNo = 0; - ui64 WritesInFly = 0; + ui64 WritesInFlight = 0; + ui64 WritesSizeInFlight = 0; ui64 OwnerPathId = 0; ui64 StatsReportRound = 0; ui64 BackgroundActivation = 0; @@ -352,7 +385,7 @@ private: TDuration FailActivationDelay = TDuration::Seconds(1); TDuration StatsReportInterval = TDuration::Seconds(10); TInstant LastAccessTime; - TInstant LastBackActivation; + TInstant LastPeriodicBackActivation; TInstant LastStatsReport; TActorId IndexingActor; // It's logically bounded to 1: we move each portion of data to multiple indices. @@ -407,10 +440,12 @@ private: bool HaveOutdatedTxs() const; bool ShardOverloaded() const { - ui64 txLimit = Settings.OverloadTxInFly; - ui64 writesLimit = Settings.OverloadWritesInFly; + ui64 txLimit = Settings.OverloadTxInFlight; + ui64 writesLimit = Settings.OverloadWritesInFlight; + ui64 writesSizeLimit = Settings.OverloadWritesSizeInFlight; return (txLimit && Executor()->GetStats().TxInFly > txLimit) || - (writesLimit && WritesInFly > writesLimit); + (writesLimit && WritesInFlight > writesLimit) || + (writesSizeLimit && WritesSizeInFlight > writesSizeLimit); } bool InsertTableOverloaded() const { @@ -430,7 +465,7 @@ private: void TryAbortWrites(NIceDb::TNiceDb& db, NOlap::TDbWrapper& dbTable, THashSet<TWriteId>&& writesToAbort); void EnqueueProgressTx(const TActorContext& ctx); - void EnqueueBackgroundActivities(bool periodic = false, bool insertOnly = false); + void EnqueueBackgroundActivities(bool periodic = false, TBackgroundActivity activity = TBackgroundActivity::All()); void UpdateSchemaSeqNo(const TMessageSeqNo& seqNo, NTabletFlatExecutor::TTransactionContext& txc); void ProtectSchemaSeqNo(const NKikimrTxColumnShard::TSchemaSeqNo& seqNoProto, NTabletFlatExecutor::TTransactionContext& txc); @@ -474,6 +509,7 @@ private: void UpdateResourceMetrics(const TActorContext& ctx, const TUsage& usage); ui64 MemoryUsage() const; void SendPeriodicStats(); + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TX_COLUMNSHARD_ACTOR; diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp index 7425597b9f..c947f54232 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp @@ -76,13 +76,21 @@ void PlanSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot } bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, ui64 metaShard, ui64 writeId, ui64 tableId, - const TString& data, std::shared_ptr<arrow::Schema> schema) { + const TString& data, std::shared_ptr<arrow::Schema> schema, bool waitResult) { const TString dedupId = ToString(writeId); auto write = std::make_unique<TEvColumnShard::TEvWrite>(sender, metaShard, writeId, tableId, dedupId, data); if (schema) { write->SetArrowSchema(NArrow::SerializeSchema(*schema)); } ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, write.release()); + + if (waitResult) { + return WaitWriteResult(runtime, metaShard) == NKikimrTxColumnShard::EResultStatus::SUCCESS; + } + return true; +} + +ui32 WaitWriteResult(TTestBasicRuntime& runtime, ui64 metaShard) { TAutoPtr<IEventHandle> handle; auto event = runtime.GrabEdgeEvent<TEvColumnShard::TEvWriteResult>(handle); UNIT_ASSERT(event); @@ -90,7 +98,7 @@ bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, ui64 metaShard, ui6 auto& resWrite = Proto(event); UNIT_ASSERT_EQUAL(resWrite.GetOrigin(), TTestTxConfig::TxTablet0); UNIT_ASSERT_EQUAL(resWrite.GetTxInitiator(), metaShard); - return (resWrite.GetStatus() == NKikimrTxColumnShard::EResultStatus::SUCCESS); + return resWrite.GetStatus(); } std::optional<ui64> WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h index 4325849b9f..f7df60895d 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/columnshard_ut_common.h @@ -378,10 +378,11 @@ bool ProposeSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, const TString void ProvideTieringSnapshot(TTestBasicRuntime& runtime, TActorId& sender, NMetadata::NFetcher::ISnapshot::TPtr snapshot); void PlanSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot snap); bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, ui64 metaShard, ui64 writeId, ui64 tableId, - const TString& data, std::shared_ptr<arrow::Schema> schema = {}); + const TString& data, std::shared_ptr<arrow::Schema> schema = {}, bool waitResult = true); std::optional<ui64> WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, ui64 tableId, const TString& dedupId, const TString& data, std::shared_ptr<arrow::Schema> schema = {}); +ui32 WaitWriteResult(TTestBasicRuntime& runtime, ui64 metaShard); void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const TVector<ui64>& pathIds, NOlap::TSnapshot snap, ui64 scanId = 0); void ProposeCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 metaShard, ui64 txId, const TVector<ui64>& writeIds); diff --git a/ydb/core/tx/columnshard/defs.h b/ydb/core/tx/columnshard/defs.h index 76253197ee..1952c9da3c 100644 --- a/ydb/core/tx/columnshard/defs.h +++ b/ydb/core/tx/columnshard/defs.h @@ -78,10 +78,12 @@ struct TCompactionLimits { struct TUsage { ui64 CPUExecTime{}; ui64 Network{}; + ui64 SourceMemorySize{}; void Add(const TUsage& other) { CPUExecTime += other.CPUExecTime; Network += other.Network; + SourceMemorySize += other.SourceMemorySize; } }; diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 9a6b973d67..30ab8d9f9b 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -36,20 +36,6 @@ struct TCompactionInfo { bool Empty() const { return Granules.empty(); } bool Good() const { return Granules.size() == 1; } - ui64 ChooseOneGranule(ui64 lastGranule) { - Y_VERIFY(Granules.size()); - - auto it = Granules.upper_bound(lastGranule); - if (it == Granules.end()) { - it = Granules.begin(); - } - - ui64 granule = *it; - Granules.clear(); - Granules.insert(granule); - return granule; - } - friend IOutputStream& operator << (IOutputStream& out, const TCompactionInfo& info) { if (info.Good() == 1) { ui64 granule = *info.Granules.begin(); @@ -258,19 +244,19 @@ struct TSelectInfo { struct TColumnEngineStats { struct TPortionsStats { - ui64 Portions{}; - ui64 Blobs{}; - ui64 Rows{}; - ui64 Bytes{}; - ui64 RawBytes{}; + i64 Portions{}; + i64 Blobs{}; + i64 Rows{}; + i64 Bytes{}; + i64 RawBytes{}; }; - ui64 Tables{}; - ui64 Granules{}; - ui64 EmptyGranules{}; - ui64 OverloadedGranules{}; - ui64 ColumnRecords{}; - ui64 ColumnMetadataBytes{}; + i64 Tables{}; + i64 Granules{}; + i64 EmptyGranules{}; + i64 OverloadedGranules{}; + i64 ColumnRecords{}; + i64 ColumnMetadataBytes{}; TPortionsStats Inserted{}; TPortionsStats Compacted{}; TPortionsStats SplitCompacted{}; @@ -287,11 +273,11 @@ struct TColumnEngineStats { }; } - ui64 ActivePortions() const { return Inserted.Portions + Compacted.Portions + SplitCompacted.Portions; } - ui64 ActiveBlobs() const { return Inserted.Blobs + Compacted.Blobs + SplitCompacted.Blobs; } - ui64 ActiveRows() const { return Inserted.Rows + Compacted.Rows + SplitCompacted.Rows; } - ui64 ActiveBytes() const { return Inserted.Bytes + Compacted.Bytes + SplitCompacted.Bytes; } - ui64 ActiveRawBytes() const { return Inserted.RawBytes + Compacted.RawBytes + SplitCompacted.RawBytes; } + i64 ActivePortions() const { return Inserted.Portions + Compacted.Portions + SplitCompacted.Portions; } + i64 ActiveBlobs() const { return Inserted.Blobs + Compacted.Blobs + SplitCompacted.Blobs; } + i64 ActiveRows() const { return Inserted.Rows + Compacted.Rows + SplitCompacted.Rows; } + i64 ActiveBytes() const { return Inserted.Bytes + Compacted.Bytes + SplitCompacted.Bytes; } + i64 ActiveRawBytes() const { return Inserted.RawBytes + Compacted.RawBytes + SplitCompacted.RawBytes; } void Clear() { *this = {}; @@ -318,7 +304,7 @@ public: const THashSet<ui32>& columnIds, std::shared_ptr<TPredicate> from, std::shared_ptr<TPredicate> to) const = 0; - virtual std::unique_ptr<TCompactionInfo> Compact() = 0; + virtual std::unique_ptr<TCompactionInfo> Compact(ui64& lastCompactedGranule) = 0; virtual std::shared_ptr<TColumnEngineChanges> StartInsert(TVector<TInsertedData>&& dataToIndex) = 0; virtual std::shared_ptr<TColumnEngineChanges> StartCompaction(std::unique_ptr<TCompactionInfo>&& compactionInfo, const TSnapshot& outdatedSnapshot) = 0; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index aa70e62ed6..de42fde260 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -489,6 +489,7 @@ void TColumnEngineForLogs::UpdatePortionStats(TColumnEngineStats& engineStats, c srcStats = &engineStats.SplitCompacted; break; case NOlap::TPortionMeta::INACTIVE: + Y_VERIFY_DEBUG(false); // Stale portions are not set INACTIVE. They have IsActive() property instead. srcStats = &engineStats.Inactive; break; case NOlap::TPortionMeta::EVICTED: @@ -779,6 +780,7 @@ std::shared_ptr<TColumnEngineChanges> TColumnEngineForLogs::StartCleanup(const T } if (affectedRecords > maxRecords) { + changes->NeedRepeat = true; break; } } @@ -818,6 +820,7 @@ std::shared_ptr<TColumnEngineChanges> TColumnEngineForLogs::StartCleanup(const T } if (affectedRecords > maxRecords) { + changes->NeedRepeat = true; break; } } @@ -1093,8 +1096,9 @@ bool TColumnEngineForLogs::ApplyChanges(IDbWrapper& db, const TChanges& changes, Y_VERIFY(!portionInfo.IsActive()); ui64 granule = portionInfo.Granule(); - if (!Granules.count(granule)) { - LOG_S_ERROR("Cannot update portion " << portionInfo << " with unknown granule at tablet " << TabletId); + ui64 portion = portionInfo.Portion(); + if (!Granules.contains(granule) || !Granules[granule]->Portions.contains(portion)) { + LOG_S_ERROR("Cannot update unknown portion " << portionInfo << " at tablet " << TabletId); return false; } @@ -1332,7 +1336,8 @@ bool TColumnEngineForLogs::ErasePortion(const TPortionInfo& portionInfo, bool ap ui64 portion = portionInfo.Portion(); if (!apply) { - if (!Granules.count(granule)) { + if (!Granules.contains(granule) || !Granules[granule]->Portions.contains(portion)) { + LOG_S_ERROR("Cannot erase unknown portion " << portionInfo << " at tablet " << TabletId); return false; } return true; @@ -1340,14 +1345,13 @@ bool TColumnEngineForLogs::ErasePortion(const TPortionInfo& portionInfo, bool ap auto& spg = Granules[granule]; Y_VERIFY(spg); - if (spg->Portions.count(portion)) { - if (updateStats) { - UpdatePortionStats(spg->Portions[portion], EStatsUpdateType::ERASE); - } - spg->Portions.erase(portion); - } else { - LOG_S_ERROR("Erase for unknown portion " << portionInfo << " at tablet " << TabletId); + Y_VERIFY(spg->Portions.contains(portion)); + + if (updateStats) { + UpdatePortionStats(spg->Portions[portion], EStatsUpdateType::ERASE); } + spg->Portions.erase(portion); + return true; // It must return true if (apply == true) } @@ -1523,44 +1527,50 @@ static bool NeedSplit(const TVector<const TPortionInfo*>& actual, const TCompact || sumSize >= limits.GranuleOverloadSize); } -std::unique_ptr<TCompactionInfo> TColumnEngineForLogs::Compact() { - auto info = std::make_unique<TCompactionInfo>(); - info->InGranule = true; - auto& out = info->Granules; +std::unique_ptr<TCompactionInfo> TColumnEngineForLogs::Compact(ui64& lastCompactedGranule) { + if (CompactionGranules.empty()) { + return {}; + } - std::vector<ui64> goodGranules; - for (ui64 granule : CompactionGranules) { + std::optional<ui64> outGranule; + bool inGranule = true; + + auto it = CompactionGranules.upper_bound(lastCompactedGranule); + while (!CompactionGranules.empty()) { + if (it == CompactionGranules.end()) { + it = CompactionGranules.begin(); + } + + ui64 granule = *it; auto spg = Granules.find(granule)->second; Y_VERIFY(spg); // We need only actual portions here (with empty XPlanStep:XTxId) auto actualPortions = GetActualPortions(spg->Portions); if (actualPortions.empty()) { + it = CompactionGranules.erase(it); continue; } ui32 inserted = 0; bool needSplit = NeedSplit(actualPortions, Limits, inserted); if (needSplit) { - if (info->InGranule) { - info->InGranule = false; - out.clear(); // clear in-granule candidates, we have a splitting one - } - out.insert(granule); + inGranule = false; + outGranule = granule; + break; } else if (inserted) { - if (info->InGranule) { - out.insert(granule); - } - } else { - goodGranules.push_back(granule); + outGranule = granule; + break; } - } - for (ui64 granule : goodGranules) { - CompactionGranules.erase(granule); + it = CompactionGranules.erase(it); } - if (!out.empty()) { + if (outGranule) { + auto info = std::make_unique<TCompactionInfo>(); + info->Granules.insert(*outGranule); + info->InGranule = inGranule; + lastCompactedGranule = *outGranule; return info; } return {}; @@ -1713,8 +1723,7 @@ SliceGranuleBatches(const TIndexInfo& indexInfo, Y_VERIFY(minTs >= ts0); // It's an estimation of needed count cause numRows calculated before key replaces - ui32 numSplitInto = changes.NumSplitInto(numRows); - ui32 rowsInGranule = numRows / numSplitInto; + const ui32 rowsInGranule = numRows / changes.NumSplitInto(numRows); Y_VERIFY(rowsInGranule); // Cannot split in case of one unique key @@ -1760,6 +1769,7 @@ SliceGranuleBatches(const TIndexInfo& indexInfo, for (auto& border : borders) { int offset = NArrow::LowerBound(keyColumn, *border.Border, batchOffsets.back()); Y_VERIFY(offset >= batchOffsets.back()); + Y_VERIFY(offset <= batch->num_rows()); batchOffsets.push_back(offset); } @@ -1770,6 +1780,7 @@ SliceGranuleBatches(const TIndexInfo& indexInfo, for (ui32 granuleNo = 0; granuleNo < borders.size() + 1; ++granuleNo) { std::vector<std::shared_ptr<arrow::RecordBatch>> granuleBatches; granuleBatches.reserve(batches.size()); + const bool lastGranule = (granuleNo == borders.size()); // Extract granule: slice source batches with offsets i64 granuleNumRows = 0; @@ -1778,14 +1789,12 @@ SliceGranuleBatches(const TIndexInfo& indexInfo, auto& batchOffsets = offsets[i]; int offset = batchOffsets[granuleNo]; - int end = batch->num_rows(); - if (granuleNo < borders.size()) { - end = batchOffsets[granuleNo + 1]; - } + int end = lastGranule ? batch->num_rows() : batchOffsets[granuleNo + 1]; int size = end - offset; Y_VERIFY(size >= 0); if (size) { + Y_VERIFY(offset < batch->num_rows()); auto slice = batch->Slice(offset, size); Y_VERIFY(slice->num_rows()); granuleNumRows += slice->num_rows(); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index d81e0122ca..e408c9283e 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -270,7 +270,7 @@ public: const THashSet<ui32>& columnIds, std::shared_ptr<TPredicate> from, std::shared_ptr<TPredicate> to) const override; - std::unique_ptr<TCompactionInfo> Compact() override; + std::unique_ptr<TCompactionInfo> Compact(ui64& lastCompactedGranule) override; // Static part of IColumnEngine iface (called from actors). It's static cause there's no threads sync. @@ -309,7 +309,7 @@ private: THashSet<ui64> GranulesInSplit; THashSet<ui64> EmptyGranules; THashMap<ui64, THashSet<ui64>> PathsGranulesOverloaded; - THashSet<ui64> CompactionGranules; + TSet<ui64> CompactionGranules; THashSet<ui64> CleanupGranules; TColumnEngineStats Counters; ui64 LastPortion; diff --git a/ydb/core/tx/columnshard/engines/portion_info.cpp b/ydb/core/tx/columnshard/engines/portion_info.cpp index b572e7cf9f..96c1bb8212 100644 --- a/ydb/core/tx/columnshard/engines/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portion_info.cpp @@ -100,7 +100,9 @@ std::shared_ptr<arrow::RecordBatch> TPortionInfo::AssembleInBatch(const TIndexIn std::shared_ptr<arrow::Table> portion = Assemble(indexInfo, schema, data); auto res = portion->CombineChunks(); Y_VERIFY(res.ok()); - return NArrow::ToBatch(portion); + auto batch = NArrow::ToBatch(portion); + Y_VERIFY(batch->Validate().ok()); + return batch; } void TPortionInfo::AddMinMax(ui32 columnId, const std::shared_ptr<arrow::Array>& column, bool sorted) { diff --git a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp index 45e95bb5cf..4cc08f41df 100644 --- a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp @@ -307,7 +307,8 @@ struct TExpected { bool Compact(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, THashMap<TBlobRange, TString>&& blobs, ui32& step, const TExpected& expected) { - auto compactionInfo = engine.Compact(); + ui64 lastCompactedGranule = 0; + auto compactionInfo = engine.Compact(lastCompactedGranule); UNIT_ASSERT_VALUES_EQUAL(compactionInfo->Granules.size(), 1); UNIT_ASSERT(!compactionInfo->InGranule); diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index 8ba7cec934..e9e7c93531 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -418,8 +418,75 @@ void TestWrite(const TestTableDescription& table) { UNIT_ASSERT(!ok); } +void TestWriteOverload(const TestTableDescription& table) { + TTestBasicRuntime runtime; + TTester::Setup(runtime); + + TActorId sender = runtime.AllocateEdgeActor(); + CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); + + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); + runtime.DispatchEvents(options); + + // + + ui64 metaShard = TTestTxConfig::TxTablet1; + ui64 writeId = 0; + ui64 tableId = 1; + + auto ydbSchema = table.Schema; + SetupSchema(runtime, sender, tableId, ydbSchema); + + TString testBlob = MakeTestBlob({0, 100 * 1000}, ydbSchema); + UNIT_ASSERT(testBlob.size() > NColumnShard::TLimits::MAX_BLOB_SIZE / 2); + UNIT_ASSERT(testBlob.size() < NColumnShard::TLimits::MAX_BLOB_SIZE); + + const ui64 overloadSize = NColumnShard::TSettings::OverloadWritesSizeInFlight; + ui32 toCatch = overloadSize / testBlob.size() + 1; + UNIT_ASSERT_VALUES_EQUAL(toCatch, 22); + TDeque<TAutoPtr<IEventHandle>> capturedWrites; + + auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { + if (auto* msg = TryGetPrivateEvent<TEvColumnShard::TEvWrite>(ev)) { + Cerr << "CATCH TEvWrite, status " << msg->PutStatus << Endl; + if (toCatch && msg->PutStatus != NKikimrProto::UNKNOWN) { + capturedWrites.push_back(ev.Release()); + --toCatch; + return true; + } else { + return false; + } + } + return false; + }; + + auto resendOneCaptured = [&]() { + UNIT_ASSERT(capturedWrites.size()); + Cerr << "RESEND TEvWrite" << Endl; + runtime.Send(capturedWrites.front().Release()); + capturedWrites.pop_front(); + }; + + runtime.SetEventFilter(captureEvents); + + const ui32 toSend = toCatch + 1; + for (ui32 i = 0; i < toSend; ++i) { + UNIT_ASSERT(WriteData(runtime, sender, metaShard, ++writeId, tableId, testBlob, {}, false)); + } + + UNIT_ASSERT_VALUES_EQUAL(WaitWriteResult(runtime, metaShard), (ui32)NKikimrTxColumnShard::EResultStatus::OVERLOADED); + + while (capturedWrites.size()) { + resendOneCaptured(); + UNIT_ASSERT_VALUES_EQUAL(WaitWriteResult(runtime, metaShard), (ui32)NKikimrTxColumnShard::EResultStatus::SUCCESS); + } + + UNIT_ASSERT(WriteData(runtime, sender, metaShard, ++writeId, tableId, testBlob)); // OK after overload +} + // TODO: Improve test. It does not catch KIKIMR-14890 -void TestWriteReadDup() { +void TestWriteReadDup(const TestTableDescription& table = {}) { TTestBasicRuntime runtime; TTester::Setup(runtime); @@ -436,7 +503,7 @@ void TestWriteReadDup() { ui64 writeId = 0; ui64 tableId = 1; - auto ydbSchema = TTestSchema::YdbSchema(); + auto ydbSchema = table.Schema; SetupSchema(runtime, sender, tableId, ydbSchema); constexpr ui32 numRows = 10; @@ -1786,6 +1853,17 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TestWrite(table); } + Y_UNIT_TEST(WriteOverload) { + TestTableDescription table; + TestWriteOverload(table); + } + + Y_UNIT_TEST(WriteStandaloneOverload) { + TestTableDescription table; + table.InStore = false; + TestWriteOverload(table); + } + Y_UNIT_TEST(WriteReadDuplicate) { TestWriteReadDup(); TestWriteReadLongTxDup(); diff --git a/ydb/core/tx/datashard/change_record.cpp b/ydb/core/tx/datashard/change_record.cpp index cba5084194..82ab36e514 100644 --- a/ydb/core/tx/datashard/change_record.cpp +++ b/ydb/core/tx/datashard/change_record.cpp @@ -12,10 +12,11 @@ #include <ydb/library/binary_json/read.h> #include <util/stream/str.h> +#include <util/string/printf.h> namespace NKikimr::NDataShard { -void TChangeRecord::SerializeTo(NKikimrChangeExchange::TChangeRecord& record) const { +void TChangeRecord::SerializeToProto(NKikimrChangeExchange::TChangeRecord& record) const { record.SetOrder(Order); record.SetGroup(Group); record.SetStep(Step); @@ -35,6 +36,12 @@ void TChangeRecord::SerializeTo(NKikimrChangeExchange::TChangeRecord& record) co } } +static auto ParseBody(const TString& protoBody) { + NKikimrChangeExchange::TChangeRecord::TDataChange body; + Y_VERIFY(body.ParseFromArray(protoBody.data(), protoBody.size())); + return body; +} + static NJson::TJsonValue StringToJson(TStringBuf in) { NJson::TJsonValue result; Y_VERIFY(NJson::ReadJsonTree(in, &result)); @@ -145,60 +152,185 @@ static void SerializeJsonValue(TUserTable::TCPtr schema, NJson::TJsonValue& valu } } -void TChangeRecord::SerializeTo(NJson::TJsonValue& json, bool virtualTimestamps) const { - switch (Kind) { - case EKind::CdcDataChange: { - Y_VERIFY(Schema); +void TChangeRecord::SerializeToYdbJson(NJson::TJsonValue& json, bool virtualTimestamps) const { + Y_VERIFY(Kind == EKind::CdcDataChange); + Y_VERIFY(Schema); - NKikimrChangeExchange::TChangeRecord::TDataChange body; - Y_VERIFY(body.ParseFromArray(Body.data(), Body.size())); + const auto body = ParseBody(Body); + SerializeJsonKey(Schema, json["key"], body.GetKey()); - SerializeJsonKey(Schema, json["key"], body.GetKey()); + if (body.HasOldImage()) { + SerializeJsonValue(Schema, json["oldImage"], body.GetOldImage()); + } - if (body.HasOldImage()) { - SerializeJsonValue(Schema, json["oldImage"], body.GetOldImage()); - } + if (body.HasNewImage()) { + SerializeJsonValue(Schema, json["newImage"], body.GetNewImage()); + } - if (body.HasNewImage()) { - SerializeJsonValue(Schema, json["newImage"], body.GetNewImage()); + const auto hasAnyImage = body.HasOldImage() || body.HasNewImage(); + switch (body.GetRowOperationCase()) { + case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: + json["update"].SetType(NJson::JSON_MAP); + if (!hasAnyImage) { + SerializeJsonValue(Schema, json["update"], body.GetUpsert()); } + break; + case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: + json["reset"].SetType(NJson::JSON_MAP); + if (!hasAnyImage) { + SerializeJsonValue(Schema, json["reset"], body.GetReset()); + } + break; + case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: + json["erase"].SetType(NJson::JSON_MAP); + break; + default: + Y_FAIL_S("Unexpected row operation: " << static_cast<int>(body.GetRowOperationCase())); + } - const auto hasAnyImage = body.HasOldImage() || body.HasNewImage(); - switch (body.GetRowOperationCase()) { - case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: - json["update"].SetType(NJson::EJsonValueType::JSON_MAP); - if (!hasAnyImage) { - SerializeJsonValue(Schema, json["update"], body.GetUpsert()); - } - break; - case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: - json["reset"].SetType(NJson::EJsonValueType::JSON_MAP); - if (!hasAnyImage) { - SerializeJsonValue(Schema, json["reset"], body.GetReset()); - } - break; - case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: - json["erase"].SetType(NJson::EJsonValueType::JSON_MAP); + if (virtualTimestamps) { + for (auto v : {Step, TxId}) { + json["ts"].AppendValue(v); + } + } +} + +static void ExtendJson(NJson::TJsonValue& value, const NJson::TJsonValue& ext) { + Y_VERIFY(ext.GetType() == NJson::JSON_MAP); + for (const auto& [k, v] : ext.GetMapSafe()) { + value.InsertValue(k, v); + } +} + +static void ToAttributeValues(TUserTable::TCPtr schema, NJson::TJsonValue& value, + const NKikimrChangeExchange::TChangeRecord::TDataChange::TSerializedCells& in) +{ + TSerializedCellVec cells; + Y_VERIFY(TSerializedCellVec::TryParse(in.GetData(), cells)); + Y_VERIFY(in.TagsSize() == cells.GetCells().size()); + + for (ui32 i = 0; i < in.TagsSize(); ++i) { + const auto tag = in.GetTags(i); + const auto& cell = cells.GetCells().at(i); + + if (cell.IsNull()) { + continue; + } + + auto it = schema->Columns.find(tag); + Y_VERIFY(it != schema->Columns.end()); + + const auto& column = it->second; + const auto& name = column.Name; + const auto type = column.Type.GetTypeId(); + + if (name == "__Hash" || name == "__CreatedAt") { + continue; // hidden column + } else if (name.StartsWith("__Hash_")) { + bool indexed = false; + for (const auto& [_, index] : schema->Indexes) { + Y_VERIFY(index.KeyColumnIds.size() >= 1); + if (index.KeyColumnIds.at(0) == tag) { + indexed = true; break; - default: - Y_FAIL_S("Unexpected row operation: " << static_cast<int>(body.GetRowOperationCase())); + } } - - if (virtualTimestamps) { - for (auto v : {Step, TxId}) { - json["ts"].AppendValue(v); + if (indexed) { + continue; // index hash column + } + } else if (name == "__RowData") { + Y_VERIFY_DEBUG(type == NScheme::NTypeIds::JsonDocument); + const auto rowData = StringToJson(NBinaryJson::SerializeToJson(cell.AsBuf())); + if (rowData.GetType() == NJson::JSON_MAP) { + auto map = rowData.GetMapSafe().find("M"); + if (map != rowData.GetMapSafe().end()) { + if (map->second.GetType() == NJson::JSON_MAP) { + ExtendJson(value, map->second); + } } } - - break; } - case EKind::AsyncIndex: { - Y_FAIL("Not supported"); + if (type == NScheme::NTypeIds::Bool) { + value.InsertValue(name, NJson::TJsonMap({{"BOOL", cell.AsValue<bool>()}})); + } else if (type == NScheme::NTypeIds::DyNumber) { + value.InsertValue(name, NJson::TJsonMap({{"N", DyNumberToString(cell.AsBuf())}})); + } else if (type == NScheme::NTypeIds::String) { + value.InsertValue(name, NJson::TJsonMap({{"B", Base64Encode(cell.AsBuf())}})); + } else if (type == NScheme::NTypeIds::Utf8) { + value.InsertValue(name, NJson::TJsonMap({{"S", cell.AsBuf()}})); } } } +void TChangeRecord::SerializeToDynamoDBStreamsJson(NJson::TJsonValue& json, const TAwsJsonOptions& opts) const { + Y_VERIFY(Kind == EKind::CdcDataChange); + Y_VERIFY(Schema); + + json = NJson::TJsonMap({ + {"awsRegion", opts.AwsRegion}, + {"dynamodb", NJson::TJsonMap({ + {"ApproximateCreationDateTime", GetApproximateCreationDateTime().MilliSeconds()}, + {"SequenceNumber", Sprintf("%0*" PRIi64, 21 /* min length */, GetSeqNo())}, + })}, + {"eventID", Sprintf("%" PRIu64 "-%" PRIi64, opts.ShardId, GetSeqNo())}, + {"eventSource", "ydb:document-table"}, + {"eventVersion", "1.0"}, + }); + + auto& dynamodb = json["dynamodb"]; + const auto body = ParseBody(Body); + + bool keysOnly = false; + bool newAndOldImages = false; + switch (opts.StreamMode) { + case TUserTable::TCdcStream::EMode::ECdcStreamModeNewImage: + dynamodb["StreamViewType"] = "NEW_IMAGE"; + break; + case TUserTable::TCdcStream::EMode::ECdcStreamModeOldImage: + dynamodb["StreamViewType"] = "OLD_IMAGE"; + break; + case TUserTable::TCdcStream::EMode::ECdcStreamModeNewAndOldImages: + dynamodb["StreamViewType"] = "NEW_AND_OLD_IMAGES"; + newAndOldImages = true; + break; + default: + dynamodb["StreamViewType"] = "KEYS_ONLY"; + keysOnly = true; + break; + } + + NJson::TJsonMap keys; + ToAttributeValues(Schema, keys, body.GetKey()); + dynamodb["Keys"] = keys; + + if (!keysOnly && body.HasOldImage()) { + ToAttributeValues(Schema, dynamodb["OldImage"], body.GetOldImage()); + ExtendJson(dynamodb["OldImage"], keys); + } + + if (!keysOnly && body.HasNewImage()) { + ToAttributeValues(Schema, dynamodb["NewImage"], body.GetNewImage()); + ExtendJson(dynamodb["NewImage"], keys); + } + + switch (body.GetRowOperationCase()) { + case NKikimrChangeExchange::TChangeRecord::TDataChange::kUpsert: + case NKikimrChangeExchange::TChangeRecord::TDataChange::kReset: + if (newAndOldImages) { + json["eventName"] = body.HasOldImage() ? "MODIFY" : "INSERT"; + } else { + json["eventName"] = "MODIFY"; + } + break; + case NKikimrChangeExchange::TChangeRecord::TDataChange::kErase: + json["eventName"] = "REMOVE"; + break; + default: + Y_FAIL_S("Unexpected row operation: " << static_cast<int>(body.GetRowOperationCase())); + } +} + TConstArrayRef<TCell> TChangeRecord::GetKey() const { if (Key) { return *Key; @@ -207,8 +339,7 @@ TConstArrayRef<TCell> TChangeRecord::GetKey() const { switch (Kind) { case EKind::AsyncIndex: case EKind::CdcDataChange: { - NKikimrChangeExchange::TChangeRecord::TDataChange parsed; - Y_VERIFY(parsed.ParseFromArray(Body.data(), Body.size())); + const auto parsed = ParseBody(Body); TSerializedCellVec key; Y_VERIFY(TSerializedCellVec::TryParse(parsed.GetKey().GetData(), key)); @@ -235,9 +366,7 @@ TString TChangeRecord::GetPartitionKey() const { switch (Kind) { case EKind::CdcDataChange: { Y_VERIFY(Schema); - - NKikimrChangeExchange::TChangeRecord::TDataChange body; - Y_VERIFY(body.ParseFromArray(Body.data(), Body.size())); + const auto body = ParseBody(Body); NJson::TJsonValue key; SerializeJsonKey(Schema, key, body.GetKey()); @@ -255,6 +384,12 @@ TString TChangeRecord::GetPartitionKey() const { return *PartitionKey; } +TInstant TChangeRecord::GetApproximateCreationDateTime() const { + return GetGroup() + ? TInstant::FromValue(GetGroup()) + : TInstant::MilliSeconds(GetStep()); +} + TString TChangeRecord::ToString() const { TString result; TStringOutput out(result); diff --git a/ydb/core/tx/datashard/change_record.h b/ydb/core/tx/datashard/change_record.h index decfa6dce6..a9b74a6549 100644 --- a/ydb/core/tx/datashard/change_record.h +++ b/ydb/core/tx/datashard/change_record.h @@ -27,6 +27,12 @@ public: CdcDataChange, }; + struct TAwsJsonOptions { + TString AwsRegion; + NKikimrSchemeOp::ECdcStreamMode StreamMode; + ui64 ShardId; + }; + public: ui64 GetOrder() const { return Order; } ui64 GetGroup() const { return Group; } @@ -41,12 +47,14 @@ public: const TPathId& GetTableId() const { return TableId; } ui64 GetSchemaVersion() const { return SchemaVersion; } - void SerializeTo(NKikimrChangeExchange::TChangeRecord& record) const; - void SerializeTo(NJson::TJsonValue& json, bool virtualTimestamps) const; + void SerializeToProto(NKikimrChangeExchange::TChangeRecord& record) const; + void SerializeToYdbJson(NJson::TJsonValue& json, bool virtualTimestamps) const; + void SerializeToDynamoDBStreamsJson(NJson::TJsonValue& json, const TAwsJsonOptions& opts) const; TConstArrayRef<TCell> GetKey() const; i64 GetSeqNo() const; TString GetPartitionKey() const; + TInstant GetApproximateCreationDateTime() const; TString ToString() const; void Out(IOutputStream& out) const; diff --git a/ydb/core/tx/datashard/change_sender_async_index.cpp b/ydb/core/tx/datashard/change_sender_async_index.cpp index 34b40d876b..e90343c601 100644 --- a/ydb/core/tx/datashard/change_sender_async_index.cpp +++ b/ydb/core/tx/datashard/change_sender_async_index.cpp @@ -118,7 +118,7 @@ class TAsyncIndexChangeSenderShard: public TActorBootstrapped<TAsyncIndexChangeS } auto& proto = *records->Record.AddRecords(); - record.SerializeTo(proto); + record.SerializeToProto(proto); Adjust(proto); } diff --git a/ydb/core/tx/datashard/change_sender_cdc_stream.cpp b/ydb/core/tx/datashard/change_sender_cdc_stream.cpp index 9b089646eb..8973479273 100644 --- a/ydb/core/tx/datashard/change_sender_cdc_stream.cpp +++ b/ydb/core/tx/datashard/change_sender_cdc_stream.cpp @@ -82,19 +82,21 @@ class TCdcChangeSenderPartition: public TActorBootstrapped<TCdcChangeSenderParti LOG_D("Handle " << ev->Get()->ToString()); NKikimrClient::TPersQueueRequest request; + const auto awsJsonOpts = TChangeRecord::TAwsJsonOptions{ + .AwsRegion = Stream.AwsRegion.GetOrElse(AppData()->AwsCompatibilityConfig.GetAwsRegion()), + .StreamMode = Stream.Mode, + .ShardId = DataShard.TabletId, + }; + for (const auto& record : ev->Get()->Records) { if (record.GetSeqNo() <= MaxSeqNo) { continue; } - const auto createdAt = record.GetGroup() - ? TInstant::FromValue(record.GetGroup()) - : TInstant::MilliSeconds(record.GetStep()); - auto& cmd = *request.MutablePartitionRequest()->AddCmdWrite(); cmd.SetSeqNo(record.GetSeqNo()); cmd.SetSourceId(NSourceIdEncoding::EncodeSimple(SourceId)); - cmd.SetCreateTimeMS(createdAt.MilliSeconds()); + cmd.SetCreateTimeMS(record.GetApproximateCreationDateTime().MilliSeconds()); cmd.SetIgnoreQuotaDeadline(true); NKikimrPQClient::TDataChunk data; @@ -103,14 +105,19 @@ class TCdcChangeSenderPartition: public TActorBootstrapped<TCdcChangeSenderParti switch (Stream.Format) { case NKikimrSchemeOp::ECdcStreamFormatProto: { NKikimrChangeExchange::TChangeRecord protoRecord; - record.SerializeTo(protoRecord); + record.SerializeToProto(protoRecord); data.SetData(protoRecord.SerializeAsString()); break; } - case NKikimrSchemeOp::ECdcStreamFormatJson: { + case NKikimrSchemeOp::ECdcStreamFormatJson: + case NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson: { NJson::TJsonValue json; - record.SerializeTo(json, Stream.VirtualTimestamps); + if (Stream.Format == NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson) { + record.SerializeToDynamoDBStreamsJson(json, awsJsonOpts); + } else { + record.SerializeToYdbJson(json, Stream.VirtualTimestamps); + } TStringStream str; NJson::TJsonWriterConfig jsonConfig; @@ -691,7 +698,8 @@ class TCdcChangeSenderMain return it->PartitionId; } - case NKikimrSchemeOp::ECdcStreamFormatJson: { + case NKikimrSchemeOp::ECdcStreamFormatJson: + case NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson: { using namespace NKikimr::NDataStreams::V1; const auto hashKey = HexBytesToDecimal(record.GetPartitionKey() /* MD5 */); return ShardFromDecimal(hashKey, KeyDesc->Partitions.size()); diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index f5e4ce2da0..8b34a53720 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -198,7 +198,7 @@ public: TDataShardEngineHost(TDataShard* self, TEngineBay& engineBay, NTable::TDatabase& db, TEngineHostCounters& counters, ui64& lockTxId, ui32& lockNodeId, TInstant now) : TEngineHost(db, counters, TEngineHostSettings(self->TabletID(), - (self->State == TShardState::Readonly || self->State == TShardState::Frozen), + (self->State == TShardState::Readonly || self->State == TShardState::Frozen || self->IsReplicated()), self->ByKeyFilterDisabled(), self->GetKeyAccessSampler())) , Self(self) diff --git a/ydb/core/tx/datashard/datashard__op_rows.cpp b/ydb/core/tx/datashard/datashard__op_rows.cpp index c55cf209f5..74610edc48 100644 --- a/ydb/core/tx/datashard/datashard__op_rows.cpp +++ b/ydb/core/tx/datashard/datashard__op_rows.cpp @@ -80,6 +80,10 @@ static void WrongShardState(NKikimrTxDataShard::TEvUploadRowsResponse& response) response.SetStatus(NKikimrTxDataShard::TError::WRONG_SHARD_STATE); } +static void ReadOnly(NKikimrTxDataShard::TEvUploadRowsResponse& response) { + response.SetStatus(NKikimrTxDataShard::TError::READONLY); +} + static void OutOfSpace(NKikimrTxDataShard::TEvEraseRowsResponse& response) { // NOTE: this function is never called, because erase is allowed when out of space response.SetStatus(NKikimrTxDataShard::TEvEraseRowsResponse::WRONG_SHARD_STATE); @@ -89,6 +93,28 @@ static void WrongShardState(NKikimrTxDataShard::TEvEraseRowsResponse& response) response.SetStatus(NKikimrTxDataShard::TEvEraseRowsResponse::WRONG_SHARD_STATE); } +static void ExecError(NKikimrTxDataShard::TEvEraseRowsResponse& response) { + response.SetStatus(NKikimrTxDataShard::TEvEraseRowsResponse::EXEC_ERROR); +} + +template <typename TEvResponse> +using TSetStatusFunc = void(*)(typename TEvResponse::ProtoRecordType&); + +template <typename TEvResponse, typename TEvRequest> +static void Reject(TDataShard* self, TEvRequest& ev, const TString& txDesc, const TString& reason, + TSetStatusFunc<TEvResponse> setStatusFunc, const TActorContext& ctx) +{ + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Rejecting " << txDesc << " request on datashard" + << ": tablet# " << self->TabletID() + << ", error# " << reason); + + auto response = MakeHolder<TEvResponse>(); + setStatusFunc(response->Record); + response->Record.SetTabletID(self->TabletID()); + response->Record.SetErrorDescription(reason); + ctx.Send(ev->Sender, std::move(response)); +} + template <typename TEvResponse, typename TEvRequest> static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& ctx, const TString& txDesc, bool isWrite) { NKikimrTxDataShard::TEvProposeTransactionResult::EStatus rejectStatus; @@ -114,19 +140,11 @@ static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& c return false; } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Rejecting " << txDesc << " request on datashard" - << ": tablet# " << self->TabletID() - << ", error# " << rejectReason); - - auto response = MakeHolder<TEvResponse>(); - response->Record.SetTabletID(self->TabletID()); if (outOfSpace) { - OutOfSpace(response->Record); + Reject<TEvResponse, TEvRequest>(self, ev, txDesc, rejectReason, &OutOfSpace, ctx); } else { - WrongShardState(response->Record); + Reject<TEvResponse, TEvRequest>(self, ev, txDesc, rejectReason, &WrongShardState, ctx); } - response->Record.SetErrorDescription(rejectReason); - ctx.Send(ev->Sender, std::move(response)); return true; } @@ -137,6 +155,10 @@ void TDataShard::Handle(TEvDataShard::TEvUploadRowsRequest::TPtr& ev, const TAct UpdateProposeQueueSize(); return; } + if (IsReplicated()) { + return Reject<TEvDataShard::TEvUploadRowsResponse>(this, ev, "bulk upsert", + "Can't execute bulk upsert at replicated table", &ReadOnly, ctx); + } if (!MaybeReject<TEvDataShard::TEvUploadRowsResponse>(this, ev, ctx, "bulk upsert", true)) { Executor()->Execute(new TTxUploadRows(this, ev), ctx); } else { @@ -150,6 +172,10 @@ void TDataShard::Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev, const TActo UpdateProposeQueueSize(); return; } + if (IsReplicated()) { + return Reject<TEvDataShard::TEvEraseRowsResponse>(this, ev, "erase", + "Can't execute erase at replicated table", &ExecError, ctx); + } if (!MaybeReject<TEvDataShard::TEvEraseRowsResponse>(this, ev, ctx, "erase", false)) { Executor()->Execute(new TTxEraseRows(this, ev), ctx); } else { diff --git a/ydb/core/tx/datashard/datashard__read_iterator.cpp b/ydb/core/tx/datashard/datashard__read_iterator.cpp index 095d731ddd..b5d62ba07b 100644 --- a/ydb/core/tx/datashard/datashard__read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard__read_iterator.cpp @@ -446,7 +446,7 @@ public: case EReadStatus::StoppedByLimit: return true; case EReadStatus::NeedData: - if (RowsRead) + if (RowsRead && CanResume()) return true; return false; } @@ -475,7 +475,7 @@ public: case EReadStatus::StoppedByLimit: return true; case EReadStatus::NeedData: - if (RowsRead) + if (RowsRead && CanResume()) return true; return false; } @@ -637,7 +637,7 @@ public: } ui64 GetRowsRead() const { return RowsRead; } - ui64 GetBytesRead() const { return BytesInResult > 0 ? BytesInResult : BlockBuilder.Bytes(); } + ui64 GetBytesRead() const { return BytesInResult > 0 ? BytesInResult : BlockBuilder.Bytes(); } bool HadInvisibleRowSkips() const { return InvisibleRowSkips > 0; } bool HadInconsistentResult() const { return HadInconsistentResult_; } @@ -645,6 +645,16 @@ public: bool NeedVolatileWaitForCommit() const { return VolatileWaitForCommit; } private: + bool CanResume() const { + if (Self->IsFollower() && State.ReadVersion.IsMax()) { + // HEAD reads from follower cannot be resumed + return false; + } + + // All other reads are assumed to be resumable + return true; + } + bool OutOfQuota() const { return RowsRead >= State.Quota.Rows || BlockBuilder.Bytes() >= State.Quota.Bytes|| @@ -656,6 +666,10 @@ private: } bool ShouldStop() { + if (!CanResume()) { + return false; + } + return OutOfQuota() || HasMaxRowsInResult() || ShouldStopByElapsedTime(); } @@ -665,11 +679,16 @@ private: NTable::TRawVals keyTo, bool reverse) { - Y_ASSERT(RowsRead < State.Quota.Rows); - Y_ASSERT(BlockBuilder.Bytes() < State.Quota.Bytes); + ui64 rowsLeft = Max<ui64>(); + ui64 bytesLeft = Max<ui64>(); + + if (CanResume()) { + Y_ASSERT(RowsRead < State.Quota.Rows); + Y_ASSERT(BlockBuilder.Bytes() < State.Quota.Bytes); - ui64 rowsLeft = State.Quota.Rows - RowsRead; - ui64 bytesLeft = State.Quota.Bytes - BlockBuilder.Bytes(); + rowsLeft = State.Quota.Rows - RowsRead; + bytesLeft = State.Quota.Bytes - BlockBuilder.Bytes(); + } auto direction = reverse ? NTable::EDirection::Reverse : NTable::EDirection::Forward; return db.Precharge(TableInfo.LocalTid, @@ -1007,32 +1026,35 @@ public: } auto& userTableInfo = it->second; - const ui64 ownerId = state.PathId.OwnerId; - TSnapshotKey snapshotKey( - ownerId, - tableId, - state.ReadVersion.Step, - state.ReadVersion.TxId); + if (!state.ReadVersion.IsMax()) { + bool snapshotFound = false; + if (!state.IsHeadRead) { + const ui64 ownerId = state.PathId.OwnerId; + TSnapshotKey snapshotKey( + ownerId, + tableId, + state.ReadVersion.Step, + state.ReadVersion.TxId); - bool isMvccVersion = state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); - bool allowMvcc = isMvccVersion && !Self->IsFollower(); - bool snapshotFound = false; - if (Self->GetSnapshotManager().FindAvailable(snapshotKey)) { - // TODO: do we need to acquire? - snapshotFound = true; - } else if (allowMvcc) { - snapshotFound = true; - } + if (Self->GetSnapshotManager().FindAvailable(snapshotKey)) { + // TODO: do we need to acquire? + snapshotFound = true; + } + } - if (!snapshotFound) { - SetStatusError( - Result->Record, - Ydb::StatusIds::PRECONDITION_FAILED, - TStringBuilder() << "Table id " << tableId << " lost snapshot at " - << state.ReadVersion << " shard " << Self->TabletID() - << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() - << (Self->IsFollower() ? " RO replica" : "")); - return EExecutionStatus::DelayComplete; + if (!snapshotFound) { + bool isMvccReadable = !Self->IsFollower() && state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); + if (!isMvccReadable) { + SetStatusError( + Result->Record, + Ydb::StatusIds::PRECONDITION_FAILED, + TStringBuilder() << "Table id " << tableId << " lost snapshot at " + << state.ReadVersion << " shard " << Self->TabletID() + << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() + << (Self->IsFollower() ? " RO replica" : "")); + return EExecutionStatus::DelayComplete; + } + } } if (state.SchemaVersion != userTableInfo->GetTableSchemaVersion()) { @@ -1094,7 +1116,7 @@ public: TDataShard::EPromotePostExecuteEdges readType = TDataShard::EPromotePostExecuteEdges::RepeatableRead; - if (state.IsHeadRead) { + if (state.IsHeadRead && !Self->IsFollower()) { bool hasError = !Result || Result->Record.HasStatus(); if (!hasError && Reader->HasUnreadQueries()) { // we failed to read all at once and also there might be dependency @@ -1228,38 +1250,50 @@ public: return; } - // we must have chosen the version - Y_VERIFY(!state.ReadVersion.IsMax()); + if (!state.ReadVersion.IsMax()) { + bool snapshotFound = false; + if (!state.IsHeadRead) { + const ui64 ownerId = state.PathId.OwnerId; + TSnapshotKey snapshotKey( + ownerId, + tableId, + state.ReadVersion.Step, + state.ReadVersion.TxId); + + if (Self->GetSnapshotManager().FindAvailable(snapshotKey)) { + // TODO: do we need to acquire? + SetUsingSnapshotFlag(); + snapshotFound = true; + } + } - const ui64 ownerId = state.PathId.OwnerId; - TSnapshotKey snapshotKey( - ownerId, - tableId, - state.ReadVersion.Step, - state.ReadVersion.TxId); + if (!snapshotFound) { + if (Self->IsFollower()) { + SetStatusError( + Result->Record, + Ydb::StatusIds::UNSUPPORTED, + TStringBuilder() << "Table id " << tableId + << " reading from snapshot " + << state.ReadVersion + << " is not supported on follower shard " << Self->TabletID()); + return; + } - bool isMvccVersion = state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); - bool allowMvcc = isMvccVersion && !Self->IsFollower(); - bool snapshotFound = false; - if (Self->GetSnapshotManager().FindAvailable(snapshotKey)) { - // TODO: do we need to acquire? - snapshotFound = true; - SetUsingSnapshotFlag(); - } else if (allowMvcc) { - snapshotFound = true; - bool isRepeatable = state.IsHeadRead ? false : true; - SetMvccSnapshot(TRowVersion(state.ReadVersion.Step, state.ReadVersion.TxId), isRepeatable); - } + bool isMvccReadable = !Self->IsFollower() && state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); + if (!isMvccReadable) { + SetStatusError( + Result->Record, + Ydb::StatusIds::PRECONDITION_FAILED, + TStringBuilder() << "Table id " << tableId << " has no snapshot at " + << state.ReadVersion << " shard " << Self->TabletID() + << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() + << (Self->IsFollower() ? " RO replica" : "")); + return; + } - if (!snapshotFound) { - SetStatusError( - Result->Record, - Ydb::StatusIds::PRECONDITION_FAILED, - TStringBuilder() << "Table id " << tableId << " has no snapshot at " - << state.ReadVersion << " shard " << Self->TabletID() - << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() - << (Self->IsFollower() ? " RO replica" : "")); - return; + bool isRepeatable = state.IsHeadRead ? false : true; + SetMvccSnapshot(TRowVersion(state.ReadVersion.Step, state.ReadVersion.TxId), isRepeatable); + } } state.SchemaVersion = userTableInfo->GetTableSchemaVersion(); @@ -1490,25 +1524,34 @@ private: TableInfo = TShortTableInfo(state.PathId.LocalPathId, *schema); } - ui64 ownerId = state.PathId.OwnerId; - TSnapshotKey snapshotKey( - ownerId, - tableId, - state.ReadVersion.Step, - state.ReadVersion.TxId); + if (!state.ReadVersion.IsMax()) { + bool snapshotFound = false; + if (!state.IsHeadRead) { + ui64 ownerId = state.PathId.OwnerId; + TSnapshotKey snapshotKey( + ownerId, + tableId, + state.ReadVersion.Step, + state.ReadVersion.TxId); - bool isMvccVersion = state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); - bool allowMvcc = isMvccVersion && !Self->IsFollower(); - if (!Self->GetSnapshotManager().FindAvailable(snapshotKey) && !allowMvcc) { - SetStatusError( - Result->Record, - Ydb::StatusIds::PRECONDITION_FAILED, - TStringBuilder() << "Table id " << tableId << " lost snapshot at " - << state.ReadVersion << " shard " << Self->TabletID() - << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() - << (Self->IsFollower() ? " RO replica" : "")); + if (Self->GetSnapshotManager().FindAvailable(snapshotKey)) { + snapshotFound = true; + } + } - return true; + if (!snapshotFound) { + bool isMvccReadable = !Self->IsFollower() && state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); + if (!isMvccReadable) { + SetStatusError( + Result->Record, + Ydb::StatusIds::PRECONDITION_FAILED, + TStringBuilder() << "Table id " << tableId << " lost snapshot at " + << state.ReadVersion << " shard " << Self->TabletID() + << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() + << (Self->IsFollower() ? " RO replica" : "")); + return true; + } + } } { @@ -1696,13 +1739,11 @@ public: << ": at tablet# " << Self->TabletID()); auto it = Self->ReadIterators.find(ReadId); - if (it == Self->ReadIterators.end()) { - // iterator aborted + if (it == Self->ReadIterators.end() && !Op) { + // iterator aborted before we could start operation return true; } - auto& state = *it->second; - try { // If tablet is in follower mode then we should sync scheme // before we build and check operation. @@ -1715,13 +1756,18 @@ public: } if (status != NKikimrTxDataShard::TError::OK) { + Y_VERIFY_DEBUG(!Op); + if (Y_UNLIKELY(it == Self->ReadIterators.end())) { + // iterator already aborted + return true; + } std::unique_ptr<TEvDataShard::TEvReadResult> result(new TEvDataShard::TEvReadResult()); SetStatusError( result->Record, Ydb::StatusIds::INTERNAL_ERROR, TStringBuilder() << "Failed to sync follower: " << errMessage); result->Record.SetReadId(ReadId.ReadId); - SendViaSession(state.SessionId, ReadId.Sender, Self->SelfId(), result.release()); + SendViaSession(it->second->SessionId, ReadId.Sender, Self->SelfId(), result.release()); return true; } @@ -1886,25 +1932,35 @@ public: TableInfo = TShortTableInfo(state.PathId.LocalPathId, *schema); } - ui64 ownerId = state.PathId.OwnerId; - TSnapshotKey snapshotKey( - ownerId, - tableId, - state.ReadVersion.Step, - state.ReadVersion.TxId); + if (!state.ReadVersion.IsMax()) { + bool snapshotFound = false; + if (!state.IsHeadRead) { + ui64 ownerId = state.PathId.OwnerId; + TSnapshotKey snapshotKey( + ownerId, + tableId, + state.ReadVersion.Step, + state.ReadVersion.TxId); - bool isMvccVersion = state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); - bool allowMvcc = isMvccVersion && !Self->IsFollower(); - if (!Self->GetSnapshotManager().FindAvailable(snapshotKey) && !allowMvcc) { - SetStatusError( - Result->Record, - Ydb::StatusIds::PRECONDITION_FAILED, - TStringBuilder() << "Table id " << tableId << " lost snapshot at " - << state.ReadVersion << " shard " << Self->TabletID() - << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() - << (Self->IsFollower() ? " RO replica" : "")); - SendResult(ctx); - return true; + if (Self->GetSnapshotManager().FindAvailable(snapshotKey)) { + snapshotFound = true; + } + } + + if (!snapshotFound) { + bool isMvccReadable = !Self->IsFollower() && state.ReadVersion >= Self->GetSnapshotManager().GetLowWatermark(); + if (!isMvccReadable) { + SetStatusError( + Result->Record, + Ydb::StatusIds::PRECONDITION_FAILED, + TStringBuilder() << "Table id " << tableId << " lost snapshot at " + << state.ReadVersion << " shard " << Self->TabletID() + << " with lowWatermark " << Self->GetSnapshotManager().GetLowWatermark() + << (Self->IsFollower() ? " RO replica" : "")); + SendResult(ctx); + return true; + } + } } { @@ -2173,27 +2229,24 @@ void TDataShard::Handle(TEvDataShard::TEvRead::TPtr& ev, const TActorContext& ct if (record.HasSnapshot()) { readVersion.Step = record.GetSnapshot().GetStep(); readVersion.TxId = record.GetSnapshot().GetTxId(); - } else if (record.GetTableId().GetOwnerId() != TabletID() && !IsFollower()) { - // sys table reads must be from HEAD, - // user tables are allowed to be read from HEAD. - - readVersion = GetMvccTxVersion(EMvccTxMode::ReadOnly, nullptr); - ev->Get()->Record.MutableSnapshot()->SetStep(readVersion.Step); - ev->Get()->Record.MutableSnapshot()->SetTxId(readVersion.TxId); - isHeadRead = true; - - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, TabletID() << " changed head to: " << readVersion.Step); + if (readVersion.IsMax()) { + replyWithError( + Ydb::StatusIds::UNSUPPORTED, + "invalid snapshot value specified"); + return; + } } if (!IsFollower()) { if (record.GetTableId().GetOwnerId() != TabletID()) { // owner is schemeshard, read user table if (readVersion.IsMax()) { - // currently not supported - replyWithError( - Ydb::StatusIds::UNSUPPORTED, - "HEAD version is unsupported"); - return; + // transform a HEAD read into some non-repeatable snapshot + readVersion = GetMvccTxVersion(EMvccTxMode::ReadOnly, nullptr); + ev->Get()->Record.MutableSnapshot()->SetStep(readVersion.Step); + ev->Get()->Record.MutableSnapshot()->SetTxId(readVersion.TxId); + isHeadRead = true; + LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, TabletID() << " changed HEAD read into non-repeatable " << readVersion); } TSnapshotKey snapshotKey( @@ -2272,21 +2325,18 @@ void TDataShard::Handle(TEvDataShard::TEvRead::TPtr& ev, const TActorContext& ct } } } else { - // follower: we can't check snapshot version, because need to sync and to sync - // we need transaction - if (readVersion.IsMax()) { - replyWithError( - Ydb::StatusIds::UNSUPPORTED, - "HEAD version on followers is unsupported"); - return; - } - if (record.GetTableId().GetOwnerId() == TabletID()) { replyWithError( Ydb::StatusIds::UNSUPPORTED, "Systable reads on followers are not supported"); return; } + + // follower: we can't check snapshot version, because need to sync and to sync + // we need transaction + if (readVersion.IsMax()) { + isHeadRead = true; + } } // Note: iterator is correct and ready to execute diff --git a/ydb/core/tx/datashard/datashard_change_receiving.cpp b/ydb/core/tx/datashard/datashard_change_receiving.cpp index eb7ab50592..b65f332735 100644 --- a/ydb/core/tx/datashard/datashard_change_receiving.cpp +++ b/ydb/core/tx/datashard/datashard_change_receiving.cpp @@ -7,12 +7,11 @@ using namespace NTabletFlatExecutor; class TDataShard::TTxChangeExchangeHandshake: public TTransactionBase<TDataShard> { using Schema = TDataShard::Schema; + static constexpr ui64 MaxBatchSize = 1000; public: - explicit TTxChangeExchangeHandshake(TDataShard* self, TEvChangeExchange::TEvHandshake::TPtr ev) + explicit TTxChangeExchangeHandshake(TDataShard* self) : TTransactionBase(self) - , Ev(std::move(ev)) - , Status(new TEvChangeExchange::TEvStatus) { } @@ -20,20 +19,59 @@ public: return TXTYPE_CHANGE_EXCHANGE_HANDSHAKE; } - bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + bool Precharge(NIceDb::TNiceDb& db) { + bool ok = true; + + ui32 i = 0; + for (const auto& [_, req] : Self->PendingChangeExchangeHandshakes) { + if (++i > MaxBatchSize) { + break; + } + + auto it = Self->InChangeSenders.find(req.GetOrigin()); + if (it == Self->InChangeSenders.end()) { + ok = ok && db.Table<Schema::ChangeSenders>().Key(req.GetOrigin()).Precharge(); + } + } + + return ok; + } + + bool Process(NIceDb::TNiceDb& db, const TActorContext& ctx) { + Statuses.reserve(Min(Self->PendingChangeExchangeHandshakes.size(), MaxBatchSize)); + + ui32 i = 0; + while (!Self->PendingChangeExchangeHandshakes.empty()) { + if (++i > MaxBatchSize) { + break; + } + + const auto& [sender, req] = Self->PendingChangeExchangeHandshakes.front(); + auto& [_, resp] = Statuses.emplace_back(sender, MakeHolder<TEvChangeExchange::TEvStatus>()); + Y_VERIFY(ExecuteHandshake(db, req, resp->Record)); + Self->PendingChangeExchangeHandshakes.pop_front(); + } + + Self->ChangeExchangeHandshakeExecuted(); + if (Self->PendingChangeExchangeHandshakes.size() < MaxBatchSize) { + Self->StartCollectingChangeExchangeHandshakes(ctx); + } else { + Self->RunChangeExchangeHandshakeTx(); + } + + return true; + } + + bool ExecuteHandshake(NIceDb::TNiceDb& db, const NKikimrChangeExchange::TEvHandshake& req, NKikimrChangeExchange::TEvStatus& resp) { if (Self->State != TShardState::Ready) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_WRONG_STATE); + resp.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + resp.SetReason(NKikimrChangeExchange::TEvStatus::REASON_WRONG_STATE); return true; } - const auto& msg = Ev->Get()->Record; - - auto it = Self->InChangeSenders.find(msg.GetOrigin()); + auto it = Self->InChangeSenders.find(req.GetOrigin()); if (it == Self->InChangeSenders.end()) { - NIceDb::TNiceDb db(txc.DB); - - auto rowset = db.Table<Schema::ChangeSenders>().Key(msg.GetOrigin()).Select(); + auto rowset = db.Table<Schema::ChangeSenders>().Key(req.GetOrigin()).Select(); if (!rowset.IsReady()) { return false; } @@ -41,38 +79,42 @@ public: if (rowset.IsValid()) { const auto generation = rowset.GetValue<Schema::ChangeSenders::Generation>(); const auto lastRecordOrder = rowset.GetValueOrDefault<Schema::ChangeSenders::LastRecordOrder>(0); - it = Self->InChangeSenders.emplace(msg.GetOrigin(), TInChangeSender(generation, lastRecordOrder)).first; + it = Self->InChangeSenders.emplace(req.GetOrigin(), TInChangeSender(generation, lastRecordOrder)).first; } else { - it = Self->InChangeSenders.emplace(msg.GetOrigin(), TInChangeSender(msg.GetGeneration())).first; + it = Self->InChangeSenders.emplace(req.GetOrigin(), TInChangeSender(req.GetGeneration())).first; } } - if (it->second.Generation > msg.GetGeneration()) { - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); - Status->Record.SetReason(NKikimrChangeExchange::TEvStatus::REASON_STALE_ORIGIN); + if (it->second.Generation > req.GetGeneration()) { + resp.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_REJECT); + resp.SetReason(NKikimrChangeExchange::TEvStatus::REASON_STALE_ORIGIN); } else { - it->second.Generation = msg.GetGeneration(); - Status->Record.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_OK); - Status->Record.SetLastRecordOrder(it->second.LastRecordOrder); - - NIceDb::TNiceDb db(txc.DB); - db.Table<Schema::ChangeSenders>().Key(it->first).Update( - NIceDb::TUpdate<Schema::ChangeSenders::Generation>(it->second.Generation), - NIceDb::TUpdate<Schema::ChangeSenders::LastSeenAt>(ctx.Now().GetValue()) - ); + it->second.Generation = req.GetGeneration(); + resp.SetStatus(NKikimrChangeExchange::TEvStatus::STATUS_OK); + resp.SetLastRecordOrder(it->second.LastRecordOrder); } return true; } + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + NIceDb::TNiceDb db(txc.DB); + + if (!Precharge(db)) { + return false; + } + + return Process(db, ctx); + } + void Complete(const TActorContext& ctx) override { - Y_VERIFY(Status); - ctx.Send(Ev->Sender, Status.Release()); + for (auto& [sender, status] : Statuses) { + ctx.Send(sender, status.Release()); + } } private: - TEvChangeExchange::TEvHandshake::TPtr Ev; - THolder<TEvChangeExchange::TEvStatus> Status; + TVector<std::pair<TActorId, THolder<TEvChangeExchange::TEvStatus>>> Statuses; }; // TTxChangeExchangeHandshake @@ -310,6 +352,12 @@ public: auto completeEdge = TRowVersion::Min(); for (const auto& record : msg.GetRecords()) { + if (record.GetOrder() <= it->second.LastRecordOrder) { + AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, + NKikimrChangeExchange::TEvStatus::REASON_ORDER_VIOLATION, + TStringBuilder() << "Last record order: " << it->second.LastRecordOrder); + break; + } if (ProcessRecord(record, txc, ctx)) { completeEdge = Max(completeEdge, TRowVersion(record.GetStep(), record.GetTxId())); } else { @@ -354,8 +402,32 @@ private: }; // TTxApplyChangeRecords +void TDataShard::StartCollectingChangeExchangeHandshakes(const TActorContext& ctx) { + if (!ChangeExchangeHandshakesCollecting) { + ChangeExchangeHandshakesCollecting = true; + ctx.Schedule(TDuration::MilliSeconds(25), new TEvPrivate::TEvChangeExchangeExecuteHandshakes); + } +} + +void TDataShard::Handle(TEvPrivate::TEvChangeExchangeExecuteHandshakes::TPtr&, const TActorContext&) { + ChangeExchangeHandshakesCollecting = false; + RunChangeExchangeHandshakeTx(); +} + +void TDataShard::RunChangeExchangeHandshakeTx() { + if (!ChangeExchangeHandshakeTxScheduled && !PendingChangeExchangeHandshakes.empty()) { + ChangeExchangeHandshakeTxScheduled = true; + EnqueueExecute(new TTxChangeExchangeHandshake(this)); + } +} + +void TDataShard::ChangeExchangeHandshakeExecuted() { + ChangeExchangeHandshakeTxScheduled = false; +} + void TDataShard::Handle(TEvChangeExchange::TEvHandshake::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxChangeExchangeHandshake(this, ev), ctx); + PendingChangeExchangeHandshakes.emplace_back(ev->Sender, std::move(ev->Get()->Record)); + StartCollectingChangeExchangeHandshakes(ctx); } void TDataShard::Handle(TEvChangeExchange::TEvApplyRecords::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 7dbd1c1a1a..5742488661 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -333,6 +333,7 @@ class TDataShard EvCdcStreamScanProgress, EvCdcStreamScanContinue, EvRestartOperation, // used to restart after an aborted scan (e.g. backup) + EvChangeExchangeExecuteHandshakes, EvEnd }; @@ -513,6 +514,8 @@ class TDataShard const ui64 TxId; }; + + struct TEvChangeExchangeExecuteHandshakes : public TEventLocal<TEvChangeExchangeExecuteHandshakes, EvChangeExchangeExecuteHandshakes> {}; }; struct Schema : NIceDb::Schema { @@ -1224,6 +1227,7 @@ class TDataShard void Handle(TEvPrivate::TEvRemoveChangeRecords::TPtr& ev, const TActorContext& ctx); // change receiving void Handle(TEvChangeExchange::TEvHandshake::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvChangeExchangeExecuteHandshakes::TPtr& ev, const TActorContext& ctx); void Handle(TEvChangeExchange::TEvApplyRecords::TPtr& ev, const TActorContext& ctx); // activation void Handle(TEvChangeExchange::TEvActivateSender::TPtr& ev, const TActorContext& ctx); @@ -1403,6 +1407,14 @@ public: return State == TShardState::Frozen; } + bool IsReplicated() const { + for (const auto& [_, info] : TableInfos) { + if (info->IsReplicated()) { + return true; + } + } + return false; + } ui32 Generation() const { return Executor()->Generation(); } bool IsFollower() const { return Executor()->GetStats().IsFollower; } @@ -2570,6 +2582,13 @@ private: // in THashMap<ui64, TInChangeSender> InChangeSenders; // ui64 is shard id + TList<std::pair<TActorId, NKikimrChangeExchange::TEvHandshake>> PendingChangeExchangeHandshakes; + bool ChangeExchangeHandshakesCollecting = false; + bool ChangeExchangeHandshakeTxScheduled = false; + + void StartCollectingChangeExchangeHandshakes(const TActorContext& ctx); + void RunChangeExchangeHandshakeTx(); + void ChangeExchangeHandshakeExecuted(); // compactionId, actorId using TCompactionWaiter = std::tuple<ui64, TActorId>; @@ -2768,6 +2787,7 @@ protected: HFunc(TEvPrivate::TEvRequestChangeRecords, Handle); HFunc(TEvPrivate::TEvRemoveChangeRecords, Handle); HFunc(TEvChangeExchange::TEvHandshake, Handle); + HFunc(TEvPrivate::TEvChangeExchangeExecuteHandshakes, Handle); HFunc(TEvChangeExchange::TEvApplyRecords, Handle); HFunc(TEvChangeExchange::TEvActivateSender, Handle); HFunc(TEvChangeExchange::TEvActivateSenderAck, Handle); diff --git a/ydb/core/tx/datashard/datashard_pipeline.cpp b/ydb/core/tx/datashard/datashard_pipeline.cpp index fa333ded45..5d5994821b 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.cpp +++ b/ydb/core/tx/datashard/datashard_pipeline.cpp @@ -1898,6 +1898,12 @@ TOperation::TPtr TPipeline::FindCompletingOp(ui64 txId) const { return nullptr; } +void TPipeline::AddCommittingOp(const TRowVersion& version) { + if (!Self->IsMvccEnabled()) + return; + CommittingOps.Add(version); +} + void TPipeline::AddCommittingOp(const TOperation::TPtr& op) { if (!Self->IsMvccEnabled() || op->IsReadOnly()) return; @@ -1909,6 +1915,12 @@ void TPipeline::AddCommittingOp(const TOperation::TPtr& op) { CommittingOps.Add(version); } +void TPipeline::RemoveCommittingOp(const TRowVersion& version) { + if (!Self->IsMvccEnabled()) + return; + CommittingOps.Remove(version); +} + void TPipeline::RemoveCommittingOp(const TOperation::TPtr& op) { if (!Self->IsMvccEnabled() || op->IsReadOnly()) return; diff --git a/ydb/core/tx/datashard/datashard_pipeline.h b/ydb/core/tx/datashard/datashard_pipeline.h index df76f64db6..12f938b681 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.h +++ b/ydb/core/tx/datashard/datashard_pipeline.h @@ -352,7 +352,9 @@ public: void RemoveCompletingOp(const TOperation::TPtr& op); TOperation::TPtr FindCompletingOp(ui64 txId) const; + void AddCommittingOp(const TRowVersion& version); void AddCommittingOp(const TOperation::TPtr& op); + void RemoveCommittingOp(const TRowVersion& version); void RemoveCommittingOp(const TOperation::TPtr& op); bool WaitCompletion(const TOperation::TPtr& op) const; bool HasCommittingOpsBelow(TRowVersion upperBound) const; diff --git a/ydb/core/tx/datashard/datashard_repl_apply.cpp b/ydb/core/tx/datashard/datashard_repl_apply.cpp index 169fd0203d..205c5528a1 100644 --- a/ydb/core/tx/datashard/datashard_repl_apply.cpp +++ b/ydb/core/tx/datashard/datashard_repl_apply.cpp @@ -9,8 +9,10 @@ using namespace NTabletFlatExecutor; class TDataShard::TTxApplyReplicationChanges : public TTransactionBase<TDataShard> { public: - explicit TTxApplyReplicationChanges(TDataShard* self, TEvDataShard::TEvApplyReplicationChanges::TPtr&& ev) + explicit TTxApplyReplicationChanges(TDataShard* self, TPipeline& pipeline, + TEvDataShard::TEvApplyReplicationChanges::TPtr&& ev) : TTransactionBase(self) + , Pipeline(pipeline) , Ev(std::move(ev)) { } @@ -22,8 +24,7 @@ public: bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { Y_UNUSED(ctx); - // TODO: check this is a replicated shard - if (Self->State != TShardState::Ready) { + if (Self->State != TShardState::Ready && !Self->IsReplicated()) { Result = MakeHolder<TEvDataShard::TEvApplyReplicationChangesResult>( NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_REJECTED, NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_WRONG_STATE); @@ -33,11 +34,11 @@ public: const auto& msg = Ev->Get()->Record; const auto& tableId = msg.GetTableId(); - TPathId pathId(tableId.GetOwnerId(), tableId.GetTableId()); + const TTableId fullTableId(tableId.GetOwnerId(), tableId.GetTableId()); const auto& userTables = Self->GetUserTables(); - auto it = userTables.find(pathId.LocalPathId); - if (pathId.OwnerId != Self->GetPathOwnerId() || it == userTables.end()) { + auto it = userTables.find(fullTableId.PathId.LocalPathId); + if (fullTableId.PathId.OwnerId != Self->GetPathOwnerId() || it == userTables.end()) { TString error = TStringBuilder() << "DataShard " << Self->TabletID() << " does not have a table " << tableId.GetOwnerId() << ":" << tableId.GetTableId(); @@ -62,17 +63,24 @@ public: return true; } - auto& source = EnsureSource(txc, pathId, msg.GetSource()); + auto& source = EnsureSource(txc, fullTableId.PathId, msg.GetSource()); for (const auto& change : msg.GetChanges()) { - if (!ApplyChange(txc, userTable, source, change)) { + if (!ApplyChange(txc, fullTableId, userTable, source, change)) { Y_VERIFY(Result); - return true; + break; } } - Result = MakeHolder<TEvDataShard::TEvApplyReplicationChangesResult>( - NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_OK); + if (MvccReadWriteVersion) { + Pipeline.AddCommittingOp(*MvccReadWriteVersion); + } + + if (!Result) { + Result = MakeHolder<TEvDataShard::TEvApplyReplicationChangesResult>( + NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_OK); + } + return true; } @@ -84,19 +92,31 @@ public: } bool ApplyChange( - TTransactionContext& txc, const TUserTable& userTable, TReplicationSourceState& source, - const NKikimrTxDataShard::TEvApplyReplicationChanges::TChange& change) + TTransactionContext& txc, const TTableId& tableId, const TUserTable& userTable, + TReplicationSourceState& source, const NKikimrTxDataShard::TEvApplyReplicationChanges::TChange& change) { + Y_VERIFY(userTable.IsReplicated()); + // TODO: check source and offset, persist new values i64 sourceOffset = change.GetSourceOffset(); ui64 writeTxId = change.GetWriteTxId(); - if (Y_UNLIKELY(writeTxId == 0)) { - Result = MakeHolder<TEvDataShard::TEvApplyReplicationChangesResult>( - NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_REJECTED, - NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_BAD_REQUEST, - "Every change must specify a non-zero WriteTxId"); - return false; + if (userTable.ReplicationConfig.HasWeakConsistency()) { + if (writeTxId) { + Result = MakeHolder<TEvDataShard::TEvApplyReplicationChangesResult>( + NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_REJECTED, + NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_BAD_REQUEST, + "WriteTxId cannot be specified for weak consistency"); + return false; + } + } else { + if (writeTxId == 0) { + Result = MakeHolder<TEvDataShard::TEvApplyReplicationChangesResult>( + NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_REJECTED, + NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_BAD_REQUEST, + "Non-zero WriteTxId must be specified for strong consistency"); + return false; + } } TSerializedCellVec keyCellVec; @@ -153,7 +173,19 @@ public: } } - txc.DB.UpdateTx(userTable.LocalTid, rop, key, update, writeTxId); + if (writeTxId) { + txc.DB.UpdateTx(userTable.LocalTid, rop, key, update, writeTxId); + } else { + if (!MvccReadWriteVersion) { + auto [readVersion, writeVersion] = Self->GetReadWriteVersions(); + Y_VERIFY_DEBUG(readVersion == writeVersion); + MvccReadWriteVersion = writeVersion; + } + + Self->SysLocksTable().BreakLocks(tableId, keyCellVec.GetCells()); + txc.DB.Update(userTable.LocalTid, rop, key, update, *MvccReadWriteVersion); + } + return true; } @@ -201,16 +233,24 @@ public: void Complete(const TActorContext& ctx) override { Y_VERIFY(Ev); Y_VERIFY(Result); - ctx.Send(Ev->Sender, Result.Release(), 0, Ev->Cookie); + + if (MvccReadWriteVersion) { + Pipeline.RemoveCommittingOp(*MvccReadWriteVersion); + Self->SendImmediateWriteResult(*MvccReadWriteVersion, Ev->Sender, Result.Release(), Ev->Cookie); + } else { + ctx.Send(Ev->Sender, Result.Release(), 0, Ev->Cookie); + } } private: + TPipeline& Pipeline; TEvDataShard::TEvApplyReplicationChanges::TPtr Ev; THolder<TEvDataShard::TEvApplyReplicationChangesResult> Result; + std::optional<TRowVersion> MvccReadWriteVersion; }; // TTxApplyReplicationChanges void TDataShard::Handle(TEvDataShard::TEvApplyReplicationChanges::TPtr& ev, const TActorContext& ctx) { - Execute(new TTxApplyReplicationChanges(this, std::move(ev)), ctx); + Execute(new TTxApplyReplicationChanges(this, Pipeline, std::move(ev)), ctx); } } // NDataShard diff --git a/ydb/core/tx/datashard/datashard_user_table.cpp b/ydb/core/tx/datashard/datashard_user_table.cpp index e357b3af61..d50c72238d 100644 --- a/ydb/core/tx/datashard/datashard_user_table.cpp +++ b/ydb/core/tx/datashard/datashard_user_table.cpp @@ -111,6 +111,16 @@ bool TUserTable::HasAsyncIndexes() const { return AsyncIndexCount > 0; } +static bool IsJsonCdcStream(TUserTable::TCdcStream::EFormat format) { + switch (format) { + case TUserTable::TCdcStream::EFormat::ECdcStreamFormatJson: + case TUserTable::TCdcStream::EFormat::ECdcStreamFormatDynamoDBStreamsJson: + return true; + default: + return false; + } +} + void TUserTable::AddCdcStream(const NKikimrSchemeOp::TCdcStreamDescription& streamDesc) { Y_VERIFY(streamDesc.HasPathId()); const auto streamPathId = PathIdFromPathId(streamDesc.GetPathId()); @@ -120,7 +130,7 @@ void TUserTable::AddCdcStream(const NKikimrSchemeOp::TCdcStreamDescription& stre } CdcStreams.emplace(streamPathId, TCdcStream(streamDesc)); - JsonCdcStreamCount += ui32(streamDesc.GetFormat() == TCdcStream::EFormat::ECdcStreamFormatJson); + JsonCdcStreamCount += ui32(IsJsonCdcStream(streamDesc.GetFormat())); NKikimrSchemeOp::TTableDescription schema; GetSchema(schema); @@ -160,7 +170,7 @@ void TUserTable::DropCdcStream(const TPathId& streamPathId) { return; } - JsonCdcStreamCount -= ui32(it->second.Format == TCdcStream::EFormat::ECdcStreamFormatJson); + JsonCdcStreamCount -= ui32(IsJsonCdcStream(it->second.Format)); CdcStreams.erase(it); NKikimrSchemeOp::TTableDescription schema; @@ -188,6 +198,15 @@ bool TUserTable::NeedSchemaSnapshots() const { return JsonCdcStreamCount > 0; } +bool TUserTable::IsReplicated() const { + switch (ReplicationConfig.Mode) { + case NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_NONE: + return false; + default: + return true; + } +} + void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) { // We expect schemeshard to send us full list of storage rooms @@ -267,6 +286,7 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) TableSchemaVersion = descr.GetTableSchemaVersion(); IsBackup = descr.GetIsBackup(); + ReplicationConfig = TReplicationConfig(descr.GetReplicationConfig()); CheckSpecialColumns(); @@ -279,7 +299,7 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) for (const auto& streamDesc : descr.GetCdcStreams()) { Y_VERIFY(streamDesc.HasPathId()); CdcStreams.emplace(PathIdFromPathId(streamDesc.GetPathId()), TCdcStream(streamDesc)); - JsonCdcStreamCount += ui32(streamDesc.GetFormat() == TCdcStream::EFormat::ECdcStreamFormatJson); + JsonCdcStreamCount += ui32(IsJsonCdcStream(streamDesc.GetFormat())); } } diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 2a5dadea1a..997ba74a9f 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -293,6 +293,7 @@ struct TUserTable : public TThrRefBase { EFormat Format; EState State; bool VirtualTimestamps = false; + TMaybe<TString> AwsRegion; TCdcStream() = default; @@ -303,6 +304,34 @@ struct TUserTable : public TThrRefBase { , State(streamDesc.GetState()) , VirtualTimestamps(streamDesc.GetVirtualTimestamps()) { + if (const auto& awsRegion = streamDesc.GetAwsRegion()) { + AwsRegion = awsRegion; + } + } + }; + + struct TReplicationConfig { + NKikimrSchemeOp::TTableReplicationConfig::EReplicationMode Mode; + NKikimrSchemeOp::TTableReplicationConfig::EConsistency Consistency; + + TReplicationConfig() + : Mode(NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_NONE) + , Consistency(NKikimrSchemeOp::TTableReplicationConfig::CONSISTENCY_UNKNOWN) + { + } + + TReplicationConfig(const NKikimrSchemeOp::TTableReplicationConfig& config) + : Mode(config.GetMode()) + , Consistency(config.GetConsistency()) + { + } + + bool HasWeakConsistency() const { + return Consistency == NKikimrSchemeOp::TTableReplicationConfig::CONSISTENCY_WEAK; + } + + bool HasStrongConsistency() const { + return Consistency == NKikimrSchemeOp::TTableReplicationConfig::CONSISTENCY_STRONG; } }; @@ -356,6 +385,7 @@ struct TUserTable : public TThrRefBase { TVector<NScheme::TTypeInfo> KeyColumnTypes; TVector<ui32> KeyColumnIds; TSerializedTableRange Range; + TReplicationConfig ReplicationConfig; bool IsBackup = false; TMap<TPathId, TTableIndex> Indexes; @@ -418,6 +448,8 @@ struct TUserTable : public TThrRefBase { bool HasCdcStreams() const; bool NeedSchemaSnapshots() const; + bool IsReplicated() const; + private: void DoApplyCreate(NTabletFlatExecutor::TTransactionContext& txc, const TString& tableName, bool shadow, const NKikimrSchemeOp::TPartitionConfig& partConfig) const; diff --git a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp index 6be8efdac3..dc5249d84c 100644 --- a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp +++ b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp @@ -2,6 +2,7 @@ #include <library/cpp/digest/md5/md5.h> #include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_writer.h> #include <ydb/core/base/path.h> #include <ydb/core/persqueue/events/global.h> @@ -715,7 +716,8 @@ Y_UNIT_TEST_SUITE(Cdc) { auto settings = TServerSettings(PortManager.GetPort(2134), {}, DefaultPQConfig()) .SetUseRealThreads(useRealThreads) .SetDomainName(root) - .SetGrpcPort(PortManager.GetPort(2135)); + .SetGrpcPort(PortManager.GetPort(2135)) + .SetEnableChangefeedDynamoDBStreamsFormat(true); Server = new TServer(settings); if (useRealThreads) { @@ -828,6 +830,11 @@ Y_UNIT_TEST_SUITE(Cdc) { return streamDesc; } + TCdcStream WithAwsRegion(const TString& awsRegion, TCdcStream streamDesc) { + streamDesc.AwsRegion = awsRegion; + return streamDesc; + } + TString CalcPartitionKey(const TString& data) { NJson::TJsonValue json; UNIT_ASSERT(NJson::ReadJsonTree(data, &json)); @@ -839,9 +846,51 @@ Y_UNIT_TEST_SUITE(Cdc) { return MD5::Calc(root.at("key").GetStringRobust()); } + static bool AreJsonsEqual(const TString& actual, const TString& expected) { + NJson::TJsonValue actualJson; + UNIT_ASSERT(NJson::ReadJsonTree(actual, &actualJson)); + NJson::TJsonValue expectedJson; + UNIT_ASSERT(NJson::ReadJsonTree(expected, &expectedJson)); + + class TScanner: public NJson::IScanCallback { + NJson::TJsonValue& Actual; + bool Success = true; + + public: + explicit TScanner(NJson::TJsonValue& actual) + : Actual(actual) + {} + + bool Do(const TString& path, NJson::TJsonValue*, NJson::TJsonValue& expectedValue) override { + if (expectedValue.GetStringRobust() != "***") { + return true; + } + + NJson::TJsonValue actualValue; + if (!Actual.GetValueByPath(path, actualValue)) { + Success = false; + return false; + } + + expectedValue = actualValue; + return true; + } + + bool IsSuccess() const { + return Success; + } + }; + + TScanner scanner(actualJson); + expectedJson.Scan(scanner); + + UNIT_ASSERT(scanner.IsSuccess()); + return actualJson == expectedJson; + } + struct PqRunner { static void Read(const TShardedTableOptions& tableDesc, const TCdcStream& streamDesc, - const TVector<TString>& queries, const TVector<TString>& records, bool strict = true) + const TVector<TString>& queries, const TVector<TString>& records, bool checkKey = true) { TTestPqEnv env(tableDesc, streamDesc); @@ -875,11 +924,9 @@ Y_UNIT_TEST_SUITE(Cdc) { pStream = data->GetPartitionStream(); for (const auto& item : data->GetMessages()) { const auto& record = records.at(reads++); - if (strict) { - UNIT_ASSERT_VALUES_EQUAL(item.GetData(), record); + UNIT_ASSERT(AreJsonsEqual(item.GetData(), record)); + if (checkKey) { UNIT_ASSERT_VALUES_EQUAL(item.GetPartitionKey(), CalcPartitionKey(record)); - } else { - UNIT_ASSERT_STRING_CONTAINS(item.GetData(), record); } } } else if (auto* create = std::get_if<TReadSessionEvent::TCreatePartitionStreamEvent>(&*ev)) { @@ -930,7 +977,7 @@ Y_UNIT_TEST_SUITE(Cdc) { struct YdsRunner { static void Read(const TShardedTableOptions& tableDesc, const TCdcStream& streamDesc, - const TVector<TString>& queries, const TVector<TString>& records, bool strict = true) + const TVector<TString>& queries, const TVector<TString>& records, bool checkKey = true) { TTestYdsEnv env(tableDesc, streamDesc); @@ -981,11 +1028,9 @@ Y_UNIT_TEST_SUITE(Cdc) { for (ui32 i = 0; i < records.size(); ++i) { const auto& actual = res.GetResult().records().at(i); const auto& expected = records.at(i); - if (strict) { - UNIT_ASSERT_VALUES_EQUAL(actual.data(), expected); + UNIT_ASSERT(AreJsonsEqual(actual.data(), expected)); + if (checkKey) { UNIT_ASSERT_VALUES_EQUAL(actual.partition_key(), CalcPartitionKey(expected)); - } else { - UNIT_ASSERT_STRING_CONTAINS(actual.data(), expected); } } } @@ -1014,7 +1059,7 @@ Y_UNIT_TEST_SUITE(Cdc) { struct TopicRunner { static void Read(const TShardedTableOptions& tableDesc, const TCdcStream& streamDesc, - const TVector<TString>& queries, const TVector<TString>& records, bool strict = true) + const TVector<TString>& queries, const TVector<TString>& records, bool checkKey = true) { TTestTopicEnv env(tableDesc, streamDesc); @@ -1047,12 +1092,8 @@ Y_UNIT_TEST_SUITE(Cdc) { pStream = data->GetPartitionSession(); for (const auto& item : data->GetMessages()) { const auto& record = records.at(reads++); - if (strict) { - UNIT_ASSERT_VALUES_EQUAL(item.GetData(), record); - // TODO: check here partition key - } else { - UNIT_ASSERT_STRING_CONTAINS(item.GetData(), record); - } + UNIT_ASSERT(AreJsonsEqual(item.GetData(), record)); + Y_UNUSED(checkKey); } } else if (auto* create = std::get_if<NYdb::NTopic::TReadSessionEvent::TStartPartitionSessionEvent>(&*ev)) { pStream = create->GetPartitionSession(); @@ -1175,10 +1216,138 @@ Y_UNIT_TEST_SUITE(Cdc) { (2, 20), (3, 30); )"}, { - R"({"update":{},"key":[1],"ts":[)", - R"({"update":{},"key":[2],"ts":[)", - R"({"update":{},"key":[3],"ts":[)", - }, false /* non-strict because of variadic timestamps */); + R"({"update":{},"key":[1],"ts":"***"})", + R"({"update":{},"key":[2],"ts":"***"})", + R"({"update":{},"key":[3],"ts":"***"})", + }); + } + + TShardedTableOptions DocApiTable() { + return TShardedTableOptions() + .Columns({ + {"__Hash", "Uint64", true, false}, + {"id_shard", "Utf8", true, false}, + {"id_sort", "Utf8", true, false}, + {"__RowData", "JsonDocument", false, false}, + {"extra", "Bool", false, false}, + }) + .Attributes({ + {"__document_api_version", "1"}, + }); + } + + Y_UNIT_TEST_TRIPLET(DocApi, PqRunner, YdsRunner, TopicRunner) { + TRunner::Read(DocApiTable(), KeysOnly(NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson), {R"( + UPSERT INTO `/Root/Table` (__Hash, id_shard, id_sort, __RowData) VALUES ( + 1, "10", "100", JsonDocument('{"M":{"color":{"S":"pink"},"weight":{"N":"4.5"}}}') + ); + )"}, { + WriteJson(NJson::TJsonMap({ + {"awsRegion", ""}, + {"dynamodb", NJson::TJsonMap({ + {"ApproximateCreationDateTime", "***"}, + {"Keys", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + })}, + {"SequenceNumber", "000000000000000000001"}, + {"StreamViewType", "KEYS_ONLY"}, + })}, + {"eventID", "***"}, + {"eventName", "MODIFY"}, + {"eventSource", "ydb:document-table"}, + {"eventVersion", "1.0"}, + }), false), + }, false /* do not check key */); + + TRunner::Read(DocApiTable(), NewAndOldImages(NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson), {R"( + UPSERT INTO `/Root/Table` (__Hash, id_shard, id_sort, __RowData, extra) VALUES ( + 1, "10", "100", JsonDocument('{"M":{"color":{"S":"pink"},"weight":{"N":"4.5"}}}'), true + ); + )", R"( + UPSERT INTO `/Root/Table` (__Hash, id_shard, id_sort, __RowData, extra) VALUES ( + 1, "10", "100", JsonDocument('{"M":{"color":{"S":"yellow"},"weight":{"N":"5.4"}}}'), false + ); + )", R"( + DELETE FROM `/Root/Table` WHERE __Hash = 1; + )"}, { + WriteJson(NJson::TJsonMap({ + {"awsRegion", ""}, + {"dynamodb", NJson::TJsonMap({ + {"ApproximateCreationDateTime", "***"}, + {"Keys", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + })}, + {"NewImage", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + {"color", NJson::TJsonMap({{"S", "pink"}})}, + {"weight", NJson::TJsonMap({{"N", "4.5"}})}, + {"extra", NJson::TJsonMap({{"BOOL", true}})}, + })}, + {"SequenceNumber", "000000000000000000001"}, + {"StreamViewType", "NEW_AND_OLD_IMAGES"}, + })}, + {"eventID", "***"}, + {"eventName", "INSERT"}, + {"eventSource", "ydb:document-table"}, + {"eventVersion", "1.0"}, + }), false), + WriteJson(NJson::TJsonMap({ + {"awsRegion", ""}, + {"dynamodb", NJson::TJsonMap({ + {"ApproximateCreationDateTime", "***"}, + {"Keys", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + })}, + {"OldImage", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + {"color", NJson::TJsonMap({{"S", "pink"}})}, + {"weight", NJson::TJsonMap({{"N", "4.5"}})}, + {"extra", NJson::TJsonMap({{"BOOL", true}})}, + })}, + {"NewImage", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + {"color", NJson::TJsonMap({{"S", "yellow"}})}, + {"weight", NJson::TJsonMap({{"N", "5.4"}})}, + {"extra", NJson::TJsonMap({{"BOOL", false}})}, + })}, + {"SequenceNumber", "000000000000000000002"}, + {"StreamViewType", "NEW_AND_OLD_IMAGES"}, + })}, + {"eventID", "***"}, + {"eventName", "MODIFY"}, + {"eventSource", "ydb:document-table"}, + {"eventVersion", "1.0"}, + }), false), + WriteJson(NJson::TJsonMap({ + {"awsRegion", ""}, + {"dynamodb", NJson::TJsonMap({ + {"ApproximateCreationDateTime", "***"}, + {"Keys", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + })}, + {"OldImage", NJson::TJsonMap({ + {"id_shard", NJson::TJsonMap({{"S", "10"}})}, + {"id_sort", NJson::TJsonMap({{"S", "100"}})}, + {"color", NJson::TJsonMap({{"S", "yellow"}})}, + {"weight", NJson::TJsonMap({{"N", "5.4"}})}, + {"extra", NJson::TJsonMap({{"BOOL", false}})}, + })}, + {"SequenceNumber", "000000000000000000003"}, + {"StreamViewType", "NEW_AND_OLD_IMAGES"}, + })}, + {"eventID", "***"}, + {"eventName", "REMOVE"}, + {"eventSource", "ydb:document-table"}, + {"eventVersion", "1.0"}, + }), false), + }, false /* do not check key */); } Y_UNIT_TEST_TRIPLET(NaN, PqRunner, YdsRunner, TopicRunner) { @@ -2197,6 +2366,52 @@ Y_UNIT_TEST_SUITE(Cdc) { }); } + Y_UNIT_TEST(AwsRegion) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetAwsRegion("defaultRegion") + .SetEnableChangefeedDynamoDBStreamsFormat(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + CreateShardedTable(server, edgeActor, "/Root", "Table", DocApiTable()); + + WaitTxNotification(server, edgeActor, AsyncAlterAddStream(server, "/Root", "Table", + KeysOnly(NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson, "Stream1"))); + WaitTxNotification(server, edgeActor, AsyncAlterAddStream(server, "/Root", "Table", + WithAwsRegion("customRegion", KeysOnly(NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson, "Stream2")))); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (__Hash, id_shard, id_sort, __RowData) VALUES ( + 1, "10", "100", JsonDocument('{"M":{"color":{"S":"pink"},"weight":{"N":"4.5"}}}') + ); + )"); + + auto checkAwsRegion = [&](const TString& path, const char* awsRegion) { + while (true) { + const auto records = GetRecords(runtime, edgeActor, path, 0); + if (records.size() >= 1) { + for (const auto& [_, record] : records) { + UNIT_ASSERT_STRING_CONTAINS(record, Sprintf(R"("awsRegion":"%s")", awsRegion)); + } + + break; + } + + SimulateSleep(server, TDuration::Seconds(1)); + } + }; + + checkAwsRegion("/Root/Table/Stream1", "defaultRegion"); + checkAwsRegion("/Root/Table/Stream2", "customRegion"); + } + } // Cdc } // NKikimr diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index d8d92b4f60..9f62807dd9 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -1048,6 +1048,7 @@ THolder<NKqp::TEvKqp::TEvQueryRequest> MakeSQLRequest(const TString &sql, request->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); request->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); } + request->Record.SetRequestType("_document_api_request"); request->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); request->Record.MutableRequest()->SetType(dml ? NKikimrKqp::QUERY_TYPE_SQL_DML @@ -1143,6 +1144,12 @@ void CreateShardedTable( } } + for (const auto& [k, v] : opts.Attributes_) { + auto* attr = tx.MutableAlterUserAttributes()->AddUserAttributes(); + attr->SetKey(k); + attr->SetValue(v); + } + desc->SetUniformPartitionsCount(opts.Shards_); if (!opts.EnableOutOfOrder_) @@ -1175,6 +1182,15 @@ void CreateShardedTable( desc->MutablePartitionConfig()->SetExecutorCacheSize(*opts.ExecutorCacheSize_); } + if (opts.Replicated_) { + desc->MutableReplicationConfig()->SetMode(NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_READ_ONLY); + } + + if (opts.ReplicationConsistency_) { + desc->MutableReplicationConfig()->SetConsistency( + static_cast<NKikimrSchemeOp::TTableReplicationConfig::EConsistency>(*opts.ReplicationConsistency_)); + } + WaitTxNotification(server, sender, RunSchemeTx(*server->GetRuntime(), std::move(request), sender)); } @@ -1647,6 +1663,8 @@ ui64 AsyncAlterAddStream( const TShardedTableOptions::TCdcStream& streamDesc) { auto request = SchemeTxTemplate(NKikimrSchemeOp::ESchemeOpCreateCdcStream, workingDir); + request->Record.SetRequestType("_document_api_request"); + auto& desc = *request->Record.MutableTransaction()->MutableModifyScheme()->MutableCreateCdcStream(); desc.SetTableName(tableName); desc.MutableStreamDescription()->SetName(streamDesc.Name); @@ -1656,6 +1674,9 @@ ui64 AsyncAlterAddStream( if (streamDesc.InitialState) { desc.MutableStreamDescription()->SetState(*streamDesc.InitialState); } + if (streamDesc.AwsRegion) { + desc.MutableStreamDescription()->SetAwsRegion(*streamDesc.AwsRegion); + } return RunSchemeTx(*server->GetRuntime(), std::move(request)); } diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h index 0340b6eb40..771704beaf 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.h +++ b/ydb/core/tx/datashard/datashard_ut_common.h @@ -395,6 +395,11 @@ enum class EShadowDataMode { Enabled, }; +enum class EReplicationConsistency: int { + Strong = 1, + Weak = 2, +}; + struct TShardedTableOptions { using TSelf = TShardedTableOptions; @@ -430,8 +435,11 @@ struct TShardedTableOptions { EFormat Format; TMaybe<EState> InitialState; bool VirtualTimestamps = false; + TMaybe<TString> AwsRegion; }; + using TAttributes = THashMap<TString, TString>; + #define TABLE_OPTION_IMPL(type, name, defaultValue) \ TSelf& name(type value) {\ name##_ = std::move(value); \ @@ -451,6 +459,9 @@ struct TShardedTableOptions { TABLE_OPTION(bool, FollowerPromotion, false); TABLE_OPTION(bool, ExternalStorage, false); TABLE_OPTION(std::optional<ui64>, ExecutorCacheSize, std::nullopt); + TABLE_OPTION(bool, Replicated, false); + TABLE_OPTION(std::optional<EReplicationConsistency>, ReplicationConsistency, std::nullopt); + TABLE_OPTION(TAttributes, Attributes, {}); #undef TABLE_OPTION #undef TABLE_OPTION_IMPL diff --git a/ydb/core/tx/datashard/datashard_ut_compaction.cpp b/ydb/core/tx/datashard/datashard_ut_compaction.cpp index 29bf2f966e..947728d89d 100644 --- a/ydb/core/tx/datashard/datashard_ut_compaction.cpp +++ b/ydb/core/tx/datashard/datashard_ut_compaction.cpp @@ -142,7 +142,10 @@ Y_UNIT_TEST_SUITE(DataShardCompaction) { InitRoot(server, sender); - CreateShardedTable(server, sender, "/Root", "table-1", 1); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions() + .Replicated(true) + .ReplicationConsistency(EReplicationConsistency::Strong) + ); auto shards1 = GetTableShards(server, sender, "/Root/table-1"); UNIT_ASSERT_VALUES_EQUAL(shards1.size(), 1u); diff --git a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp index fc453a4d08..54167ea4ba 100644 --- a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp @@ -94,8 +94,8 @@ TVector<TString> SerializeKeys(const TVector<ui32>& keys, TKeyCellsMaker makeKey TProto::TEvEraseRequest MakeEraseRowsRequest( const TTableId& tableId, const TVector<ui32>& keyTags, - const TVector<TString>& keys) { - + const TVector<TString>& keys) +{ TProto::TEvEraseRequest request; request.SetTableId(tableId.PathId.LocalPathId); @@ -180,7 +180,8 @@ private: void EraseRows( TServer::TPtr server, const TActorId& sender, const TString& path, const TTableId& tableId, TVector<ui32> keyTags, TVector<TString> keys, - ui32 status = TProto::TEvEraseResponse::OK, const TString& error = "") { + ui32 status = TProto::TEvEraseResponse::OK, const TString& error = "") +{ using TEvRequest = TEvDataShard::TEvEraseRowsRequest; using TEvResponse = TEvDataShard::TEvEraseRowsResponse; @@ -772,6 +773,27 @@ key = 4, value = (empty maybe) auto content = ReadShardedTable(server, "/Root/table-1"); UNIT_ASSERT_STRINGS_EQUAL(StripInPlace(content), "key = 3, value = 2030-04-15T00:00:00.000000Z"); } + + Y_UNIT_TEST(EraseRowsFromReplicatedTable) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings + .SetDomainName("Root") + .SetUseRealThreads(false); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + const TActorId sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + + InitRoot(server, sender); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions().Replicated(true)); + + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + EraseRows(server, sender, "/Root/table-1", tableId, {1}, SerializeKeys({1, 2}), + TProto::TEvEraseResponse::EXEC_ERROR, "Can't execute erase at replicated table"); + } } Y_UNIT_TEST_SUITE(DistributedEraseTests) { diff --git a/ydb/core/tx/datashard/datashard_ut_followers.cpp b/ydb/core/tx/datashard/datashard_ut_followers.cpp index ea21a77677..418e6ab1af 100644 --- a/ydb/core/tx/datashard/datashard_ut_followers.cpp +++ b/ydb/core/tx/datashard/datashard_ut_followers.cpp @@ -89,6 +89,82 @@ Y_UNIT_TEST_SUITE(DataShardFollowers) { } } + Y_UNIT_TEST(FollowerStaleRo) { + const bool useSource = true; + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + NKikimrConfig::TAppConfig appCfg; + appCfg.MutableTableServiceConfig()->SetEnableKqpDataQuerySourceRead(useSource); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false) + .SetEnableForceFollowers(true) + .SetAppConfig(appCfg); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); + + InitRoot(server, sender); + + CreateShardedTable(server, sender, "/Root", "table-1", + TShardedTableOptions() + .Shards(2) + .Followers(1)); + + ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (2, 2), (3, 3);"); + + bool dropFollowerUpdates = true; + + { + auto result = KqpSimpleStaleRoExec(runtime, "SELECT * FROM `/Root/table-1`", "/Root"); + TString expected = "{ items { uint32_value: 1 } items { uint32_value: 1 } }, " + "{ items { uint32_value: 2 } items { uint32_value: 2 } }, " + "{ items { uint32_value: 3 } items { uint32_value: 3 } }"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + std::vector<TAutoPtr<IEventHandle>> capturedUpdates; + auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { + if (ev->GetTypeRewrite() == NKikimr::TEvTablet::TEvFollowerUpdate::EventType || + ev->GetTypeRewrite() == NKikimr::TEvTablet::TEvFollowerAuxUpdate::EventType || + ev->GetTypeRewrite() == NKikimr::TEvTablet::TEvFUpdate::EventType || + ev->GetTypeRewrite() == NKikimr::TEvTablet::TEvFAuxUpdate::EventType) + { + + if (dropFollowerUpdates) { + capturedUpdates.emplace_back(ev); + return true; + } + Cerr << "Followers update " << capturedUpdates.size() << Endl; + } + + return false; + }; + + // blocking followers from new log updates. + runtime.SetEventFilter(captureEvents); + + ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 4), (2, 5), (3, 6);"); + + { + auto result = KqpSimpleStaleRoExec(runtime, "SELECT * FROM `/Root/table-1` where key = 1", "/Root"); + TString expected = "{ items { uint32_value: 1 } items { uint32_value: 1 } }"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + + { + // multiple shards, always read from main tablets. + auto result = KqpSimpleStaleRoExec(runtime, "SELECT * FROM `/Root/table-1`", "/Root"); + TString expected = "{ items { uint32_value: 1 } items { uint32_value: 4 } }, " + "{ items { uint32_value: 2 } items { uint32_value: 5 } }, " + "{ items { uint32_value: 3 } items { uint32_value: 6 } }"; + UNIT_ASSERT_VALUES_EQUAL(result, expected); + } + } + } // Y_UNIT_TEST_SUITE(DataShardFollowers) } // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp index c15c674b25..81c7efd31c 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp @@ -5,6 +5,7 @@ #include <ydb/core/formats/arrow_helpers.h> #include <ydb/core/kqp/ut/common/kqp_ut_common.h> +#include <ydb/core/tablet_flat/shared_cache_events.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/tx_proxy/read_table.h> @@ -2289,10 +2290,10 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); const auto& record = readResult->Record; - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus().GetCode(), Ydb::StatusIds::PRECONDITION_FAILED); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus().GetCode(), Ydb::StatusIds::UNSUPPORTED); } - Y_UNIT_TEST(ShouldNotReadHeadFromFollower) { + Y_UNIT_TEST(ShouldReadHeadFromFollower) { TPortManager pm; TServerSettings serverSettings(pm.GetPort(2134)); serverSettings.SetDomainName("Root") @@ -2301,13 +2302,14 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { const ui64 shardCount = 1; TTestHelper helper(serverSettings, shardCount, true); - TRowVersion someVersion = TRowVersion(10000, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, someVersion); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, TRowVersion::Max()); request->Record.ClearSnapshot(); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); - const auto& record = readResult->Record; - UNIT_ASSERT_VALUES_EQUAL(record.GetStatus().GetCode(), Ydb::StatusIds::UNSUPPORTED); + + CheckResult(helper.Tables["table-1"].UserTable, *readResult, { + {3, 3, 3, 300}, + }); } Y_UNIT_TEST(ShouldStopWhenNodeDisconnected) { @@ -3539,4 +3541,101 @@ Y_UNIT_TEST_SUITE(DataShardReadIteratorState) { } }; +Y_UNIT_TEST_SUITE(DataShardReadIteratorPageFaults) { + Y_UNIT_TEST(CancelPageFaultedReadThenDropTable) { + TPortManager pm; + NFake::TCaches caches; + caches.Shared = 1 /* bytes */; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false) + .SetCacheParams(caches); + TServer::TPtr server = new TServer(serverSettings); + + auto& runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_NOTICE); + runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_INFO); + // runtime.SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NLog::PRI_DEBUG); + + InitRoot(server, sender); + + TDisableDataShardLogBatching disableDataShardLogBatching; + + auto opts = TShardedTableOptions() + .Shards(1) + .ExecutorCacheSize(1 /* byte */) + .Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false}}); + CreateShardedTable(server, sender, "/Root", "table-1", opts); + + ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)")); + SimulateSleep(runtime, TDuration::Seconds(1)); + + const auto shard1 = GetTableShards(server, sender, "/Root/table-1").at(0); + const auto tableId1 = ResolveTableId(server, sender, "/Root/table-1"); + CompactTable(runtime, shard1, tableId1, false); + RebootTablet(runtime, shard1, sender); + SimulateSleep(runtime, TDuration::Seconds(1)); + + size_t observedReadResults = 0; + bool captureCacheRequests = true; + std::vector<std::unique_ptr<IEventHandle>> capturedCacheRequests; + auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) -> auto { + switch (ev->GetTypeRewrite()) { + case TEvDataShard::TEvReadResult::EventType: { + auto* msg = ev->Get<TEvDataShard::TEvReadResult>(); + Cerr << "... observed TEvReadResult:\n" << msg->ToString() << Endl; + observedReadResults++; + break; + } + case NSharedCache::TEvRequest::EventType: { + if (captureCacheRequests) { + Cerr << "... captured TEvRequest" << Endl; + capturedCacheRequests.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + break; + } + } + return TTestActorRuntime::EEventAction::PROCESS; + }; + auto prevObserverFunc = runtime.SetObserverFunc(captureEvents); + + auto readSender = runtime.AllocateEdgeActor(); + auto tabletPipe = runtime.ConnectToPipe(shard1, readSender, 0, NTabletPipe::TClientConfig()); + { + auto request = std::make_unique<TEvDataShard::TEvRead>(); + request->Record.SetReadId(1); + request->Record.MutableTableId()->SetOwnerId(tableId1.PathId.OwnerId); + request->Record.MutableTableId()->SetTableId(tableId1.PathId.LocalPathId); + request->Record.MutableTableId()->SetSchemaVersion(tableId1.SchemaVersion); + request->Record.AddColumns(1); + request->Record.AddColumns(2); + request->Ranges.emplace_back(TOwnedCellVec(), true, TOwnedCellVec(), true); + runtime.SendToPipe(tabletPipe, readSender, request.release()); + } + + WaitFor(runtime, [&]() { return capturedCacheRequests.size() > 0 || observedReadResults > 0; }, "shared cache request"); + UNIT_ASSERT_C(capturedCacheRequests.size() > 0, "cache request was not captured"); + + { + auto request = std::make_unique<TEvDataShard::TEvReadCancel>(); + request->Record.SetReadId(1); + runtime.SendToPipe(tabletPipe, readSender, request.release()); + } + SimulateSleep(runtime, TDuration::Seconds(1)); + + captureCacheRequests = false; + for (auto& ev : capturedCacheRequests) { + runtime.Send(ev.release(), 0, true); + } + + // We should be able to drop table + WaitTxNotification(server, AsyncDropTable(server, sender, "/Root", "table-1")); + } +} + } // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_ut_replication.cpp b/ydb/core/tx/datashard/datashard_ut_replication.cpp index ef98ed6619..00b78a2789 100644 --- a/ydb/core/tx/datashard/datashard_ut_replication.cpp +++ b/ydb/core/tx/datashard/datashard_ut_replication.cpp @@ -26,8 +26,14 @@ Y_UNIT_TEST_SUITE(DataShardReplication) { InitRoot(server, sender); - CreateShardedTable(server, sender, "/Root", "table-1", 1); - CreateShardedTable(server, sender, "/Root", "table-2", 1); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions() + .Replicated(true) + .ReplicationConsistency(EReplicationConsistency::Strong) + ); + CreateShardedTable(server, sender, "/Root", "table-2", TShardedTableOptions() + .Replicated(true) + .ReplicationConsistency(EReplicationConsistency::Strong) + ); auto shards1 = GetTableShards(server, sender, "/Root/table-1"); auto shards2 = GetTableShards(server, sender, "/Root/table-2"); @@ -96,8 +102,14 @@ Y_UNIT_TEST_SUITE(DataShardReplication) { InitRoot(server, sender); - CreateShardedTable(server, sender, "/Root", "table-1", 1); - CreateShardedTable(server, sender, "/Root", "table-2", 1); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions() + .Replicated(true) + .ReplicationConsistency(EReplicationConsistency::Strong) + ); + CreateShardedTable(server, sender, "/Root", "table-2", TShardedTableOptions() + .Replicated(true) + .ReplicationConsistency(EReplicationConsistency::Strong) + ); auto shards1 = GetTableShards(server, sender, "/Root/table-1"); auto tableId1 = ResolveTableId(server, sender, "/Root/table-1"); @@ -214,6 +226,61 @@ Y_UNIT_TEST_SUITE(DataShardReplication) { DoSplitMergeChanges(true); } + Y_UNIT_TEST(ReplicatedTable) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + + InitRoot(server, sender); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions().Replicated(true)); + + ExecSQL(server, sender, "SELECT * FROM `/Root/table-1`"); + ExecSQL(server, sender, "INSERT INTO `/Root/table-1` (key, value) VALUES (1, 10);", true, + Ydb::StatusIds::GENERIC_ERROR); + } + + Y_UNIT_TEST(ApplyChangesToReplicatedTable) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + + InitRoot(server, sender); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions() + .Replicated(true) + .ReplicationConsistency(EReplicationConsistency::Weak) + ); + + auto shards = GetTableShards(server, sender, "/Root/table-1"); + auto tableId = ResolveTableId(server, sender, "/Root/table-1"); + + ApplyChanges(server, shards.at(0), tableId, "my-source", { + TChange{ .Offset = 0, .WriteTxId = 0, .Key = 1, .Value = 11 }, + TChange{ .Offset = 1, .WriteTxId = 0, .Key = 2, .Value = 22 }, + TChange{ .Offset = 2, .WriteTxId = 0, .Key = 3, .Value = 33 }, + }); + + auto result = ReadShardedTable(server, "/Root/table-1"); + UNIT_ASSERT_VALUES_EQUAL(result, + "key = 1, value = 11\n" + "key = 2, value = 22\n" + "key = 3, value = 33\n" + ); + } + } } // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp b/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp index ce54103122..d826ff4e91 100644 --- a/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_upload_rows.cpp @@ -726,6 +726,24 @@ Y_UNIT_TEST_SUITE(TTxDataShardUploadRows) { "key = 10, value = (empty maybe), extra = (empty maybe)\n"); } + Y_UNIT_TEST(UploadRowsToReplicatedTable) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); + + InitRoot(server, sender); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions().Replicated(true)); + + DoUploadTestRows(server, sender, "/Root/table-1", Ydb::Type::UINT32, Ydb::StatusIds::GENERIC_ERROR); + } + } } // namespace NKikimr diff --git a/ydb/core/tx/datashard/execution_unit.cpp b/ydb/core/tx/datashard/execution_unit.cpp index 5e8c32a658..404bfd25ea 100644 --- a/ydb/core/tx/datashard/execution_unit.cpp +++ b/ydb/core/tx/datashard/execution_unit.cpp @@ -216,6 +216,19 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& return true; } + if (DataShard.IsReplicated() && !(op->IsReadOnly() || op->IsCommitWritesTx())) { + TString err = TStringBuilder() + << "Can't execute write tx at replicated table:" + << " tablet id: " << DataShard.TabletID(); + BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR) + ->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, err); + + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + + op->Abort(); + return true; + } + return false; } @@ -248,6 +261,10 @@ bool TExecutionUnit::WillRejectDataTx(TOperation::TPtr op) const { return true; } + if (DataShard.IsReplicated() && !(op->IsReadOnly() || op->IsCommitWritesTx())) { + return true; + } + return false; } diff --git a/ydb/core/tx/schemeshard/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/CMakeLists.darwin.txt index 2c2f31c9bd..077d0420fa 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.darwin.txt @@ -60,6 +60,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC contrib-libs-protobuf cpp-deprecated-enum_codegen cpp-html-pcdata + library-cpp-json ydb-core-actorlib_impl ydb-core-audit ydb-core-base diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt index 6e714991fc..969a4ee710 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt @@ -61,6 +61,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC contrib-libs-protobuf cpp-deprecated-enum_codegen cpp-html-pcdata + library-cpp-json ydb-core-actorlib_impl ydb-core-audit ydb-core-base diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/CMakeLists.linux.txt index 6e714991fc..969a4ee710 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux.txt @@ -61,6 +61,7 @@ target_link_libraries(core-tx-schemeshard PUBLIC contrib-libs-protobuf cpp-deprecated-enum_codegen cpp-html-pcdata + library-cpp-json ydb-core-actorlib_impl ydb-core-audit ydb-core-base diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 6447ebd13f..539320d087 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -327,7 +327,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui32, ui64, TString, TString, TString, ui64, TString, bool> TTableRec; + typedef std::tuple<TPathId, ui32, ui64, TString, TString, TString, ui64, TString, bool, TString> TTableRec; typedef TDeque<TTableRec> TTableRows; bool LoadTables(NIceDb::TNiceDb& db, TTableRows& tableRows) const { @@ -347,9 +347,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui64 partitionVersion = rowSet.GetValueOrDefault<Schema::Tables::PartitioningVersion>(0); TString ttlSettings = rowSet.GetValueOrDefault<Schema::Tables::TTLSettings>(); bool isBackup = rowSet.GetValueOrDefault<Schema::Tables::IsBackup>(false); + TString replConfig = rowSet.GetValueOrDefault<Schema::Tables::ReplicationConfig>(); tableRows.emplace_back(pathId, - nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, partitionVersion, ttlSettings, isBackup); + nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, + partitionVersion, ttlSettings, isBackup, replConfig); if (!rowSet.Next()) { return false; @@ -375,9 +377,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui64 partitionVersion = rowSet.GetValueOrDefault<Schema::MigratedTables::PartitioningVersion>(0); TString ttlSettings = rowSet.GetValueOrDefault<Schema::MigratedTables::TTLSettings>(); bool isBackup = rowSet.GetValueOrDefault<Schema::MigratedTables::IsBackup>(false); + TString replConfig = rowSet.GetValueOrDefault<Schema::MigratedTables::ReplicationConfig>(); tableRows.emplace_back(pathId, - nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, partitionVersion, ttlSettings, isBackup); + nextCollId, alterVersion, partitionConfig, alterTabletFull, alterTabletDiff, + partitionVersion, ttlSettings, isBackup, replConfig); if (!rowSet.Next()) { return false; @@ -1323,6 +1327,29 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } + template <typename TRowSet> + TSchemeLimits LoadSchemeLimits(const TSchemeLimits& defaults, TRowSet& rowSet) { + return TSchemeLimits { + .MaxDepth = rowSet.template GetValueOrDefault<Schema::SubDomains::DepthLimit>(defaults.MaxDepth), + .MaxPaths = rowSet.template GetValueOrDefault<Schema::SubDomains::PathsLimit>(defaults.MaxPaths), + .MaxChildrenInDir = rowSet.template GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(defaults.MaxChildrenInDir), + .MaxAclBytesSize = rowSet.template GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(defaults.MaxAclBytesSize), + .MaxPathElementLength = rowSet.template GetValueOrDefault<Schema::SubDomains::PathElementLength>(defaults.MaxPathElementLength), + .ExtraPathSymbolsAllowed = rowSet.template GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(defaults.ExtraPathSymbolsAllowed), + .MaxTableColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(defaults.MaxTableColumns), + .MaxTableColumnNameLength = rowSet.template GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(defaults.MaxTableColumnNameLength), + .MaxTableKeyColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(defaults.MaxTableKeyColumns), + .MaxTableIndices = rowSet.template GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(defaults.MaxTableIndices), + .MaxTableCdcStreams = rowSet.template GetValueOrDefault<Schema::SubDomains::TableCdcStreamsLimit>(defaults.MaxTableCdcStreams), + .MaxShards = rowSet.template GetValueOrDefault<Schema::SubDomains::ShardsLimit>(defaults.MaxShards), + .MaxShardsInPath = rowSet.template GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(defaults.MaxShardsInPath), + .MaxConsistentCopyTargets = rowSet.template GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(defaults.MaxConsistentCopyTargets), + .MaxPQPartitions = rowSet.template GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(defaults.MaxPQPartitions), + .MaxExports = rowSet.template GetValueOrDefault<Schema::SubDomains::ExportsLimit>(defaults.MaxExports), + .MaxImports = rowSet.template GetValueOrDefault<Schema::SubDomains::ImportsLimit>(defaults.MaxImports), + }; + } + bool ReadEverything(TTransactionContext& txc, const TActorContext& ctx) { const TOwnerId selfId = Self->TabletID(); @@ -1535,21 +1562,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { if (row.IsValid()) { version = row.GetValue<Schema::SubDomains::AlterVersion>(); - rootLimits.MaxDepth = row.GetValueOrDefault<Schema::SubDomains::DepthLimit>(rootLimits.MaxDepth); - rootLimits.MaxPaths = row.GetValueOrDefault<Schema::SubDomains::PathsLimit>(rootLimits.MaxPathsCompat); - rootLimits.MaxChildrenInDir = row.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(rootLimits.MaxChildrenInDir); - rootLimits.MaxAclBytesSize = row.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(rootLimits.MaxAclBytesSize); - rootLimits.MaxTableColumns = row.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(rootLimits.MaxTableColumns); - rootLimits.MaxTableColumnNameLength = row.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(rootLimits.MaxTableColumnNameLength); - rootLimits.MaxTableKeyColumns = row.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(rootLimits.MaxTableKeyColumns); - rootLimits.MaxTableIndices = row.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(rootLimits.MaxTableIndices); - rootLimits.MaxTableCdcStreams = row.GetValueOrDefault<Schema::SubDomains::TableCdcStreamsLimit>(rootLimits.MaxTableCdcStreams); - rootLimits.MaxShards = row.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(rootLimits.MaxShards); - rootLimits.MaxShardsInPath = row.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(rootLimits.MaxShardsInPath); - rootLimits.MaxConsistentCopyTargets = row.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(rootLimits.MaxConsistentCopyTargets); - rootLimits.MaxPathElementLength = row.GetValueOrDefault<Schema::SubDomains::PathElementLength>(rootLimits.MaxPathElementLength); - rootLimits.ExtraPathSymbolsAllowed = row.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(rootLimits.ExtraPathSymbolsAllowed); - rootLimits.MaxPQPartitions = row.GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(rootLimits.MaxPQPartitions); + rootLimits.MaxPaths = rootLimits.MaxPathsCompat; + rootLimits = LoadSchemeLimits(rootLimits, row); } TSubDomainInfo::TPtr rootDomainInfo = new TSubDomainInfo(version, Self->RootPathId()); @@ -1593,24 +1607,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TTabletId sharedHiveId = rowset.GetValue<Schema::SubDomains::SharedHiveId>(); domainInfo->SetSharedHive(sharedHiveId); - TSchemeLimits limits = rootLimits; - limits.MaxDepth = rowset.GetValueOrDefault<Schema::SubDomains::DepthLimit>(limits.MaxDepth); - limits.MaxPaths = rowset.GetValueOrDefault<Schema::SubDomains::PathsLimit>(limits.MaxPaths); - limits.MaxChildrenInDir = rowset.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(limits.MaxChildrenInDir); - limits.MaxAclBytesSize = rowset.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(limits.MaxAclBytesSize); - limits.MaxTableColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(limits.MaxTableColumns); - limits.MaxTableColumnNameLength = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(limits.MaxTableColumnNameLength); - limits.MaxTableKeyColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(limits.MaxTableKeyColumns); - limits.MaxTableIndices = rowset.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(limits.MaxTableIndices); - limits.MaxTableCdcStreams = rowset.GetValueOrDefault<Schema::SubDomains::TableCdcStreamsLimit>(limits.MaxTableCdcStreams); - limits.MaxShards = rowset.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(limits.MaxShards); - limits.MaxShardsInPath = rowset.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(limits.MaxShardsInPath); - limits.MaxConsistentCopyTargets = rowset.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(limits.MaxConsistentCopyTargets); - limits.MaxPathElementLength = rowset.GetValueOrDefault<Schema::SubDomains::PathElementLength>(limits.MaxPathElementLength); - limits.ExtraPathSymbolsAllowed = rowset.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(limits.ExtraPathSymbolsAllowed); - limits.MaxPQPartitions = rowset.GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(limits.MaxPQPartitions); - - domainInfo->SetSchemeLimits(limits); + domainInfo->SetSchemeLimits(LoadSchemeLimits(rootLimits, rowset)); if (rowset.HaveValue<Schema::SubDomains::DeclaredSchemeQuotas>()) { NKikimrSubDomains::TSchemeQuotas declaredSchemeQuotas; @@ -1860,12 +1857,16 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableInfo->PartitioningVersion = std::get<6>(rec); - TString ttlSettings = std::get<7>(rec); - if (ttlSettings) { + if (const auto ttlSettings = std::get<7>(rec)) { bool parseOk = ParseFromStringNoSizeLimit(tableInfo->MutableTTLSettings(), ttlSettings); Y_VERIFY(parseOk); } + if (const auto replicationConfig = std::get<9>(rec)) { + bool parseOk = ParseFromStringNoSizeLimit(tableInfo->MutableReplicationConfig(), replicationConfig); + Y_VERIFY(parseOk); + } + tableInfo->IsBackup = std::get<8>(rec); Self->Tables[pathId] = tableInfo; @@ -2845,6 +2846,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto mode = rowset.GetValue<Schema::CdcStream::Mode>(); auto format = rowset.GetValue<Schema::CdcStream::Format>(); auto vt = rowset.GetValueOrDefault<Schema::CdcStream::VirtualTimestamps>(false); + auto awsRegion = rowset.GetValue<Schema::CdcStream::AwsRegion>(); auto state = rowset.GetValue<Schema::CdcStream::State>(); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); @@ -2855,7 +2857,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { << ", path type: " << NKikimrSchemeOp::EPathType_Name(path->PathType)); Y_VERIFY(!Self->CdcStreams.contains(pathId)); - Self->CdcStreams[pathId] = new TCdcStreamInfo(alterVersion, mode, format, vt, state); + Self->CdcStreams[pathId] = new TCdcStreamInfo(alterVersion, mode, format, vt, awsRegion, state); Self->IncrementPathDbRefCount(pathId); if (state == NKikimrSchemeOp::ECdcStreamStateScan) { @@ -2888,6 +2890,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto mode = rowset.GetValue<Schema::CdcStreamAlterData::Mode>(); auto format = rowset.GetValue<Schema::CdcStreamAlterData::Format>(); auto vt = rowset.GetValueOrDefault<Schema::CdcStreamAlterData::VirtualTimestamps>(false); + auto awsRegion = rowset.GetValue<Schema::CdcStreamAlterData::AwsRegion>(); auto state = rowset.GetValue<Schema::CdcStreamAlterData::State>(); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); @@ -2899,14 +2902,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { if (!Self->CdcStreams.contains(pathId)) { Y_VERIFY(alterVersion == 1); - Self->CdcStreams[pathId] = TCdcStreamInfo::New(mode, format, vt); + Self->CdcStreams[pathId] = TCdcStreamInfo::New(mode, format, vt, awsRegion); Self->IncrementPathDbRefCount(pathId); } auto stream = Self->CdcStreams.at(pathId); Y_VERIFY(stream->AlterData == nullptr); Y_VERIFY(stream->AlterVersion < alterVersion); - stream->AlterData = new TCdcStreamInfo(alterVersion, mode, format, vt, state); + stream->AlterData = new TCdcStreamInfo(alterVersion, mode, format, vt, awsRegion, state); Y_VERIFY_S(Self->PathsById.contains(path->ParentPathId), "Parent path is not found" << ", cdc stream pathId: " << pathId diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp index 47247d1f95..6d37d4f2e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp @@ -382,7 +382,7 @@ struct TSchemeShard::TTxInitTenantSchemeShard : public TSchemeShard::TRwTxBase { subdomain->AddStoragePool(x); } - subdomain->SetSchemeLimits(TSchemeLimits(schemeLimits)); + subdomain->SetSchemeLimits(TSchemeLimits::FromProto(schemeLimits)); if (record.HasDeclaredSchemeQuotas()) { subdomain->ApplyDeclaredSchemeQuotas(record.GetDeclaredSchemeQuotas(), ctx.Now()); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp index 754f1efe4f..919af110be 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp @@ -138,6 +138,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderOperation(); @@ -364,6 +365,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderDeleting() .NotUnderOperation(); @@ -508,6 +510,7 @@ TVector<ISubOperationBase::TPtr> CreateAlterCdcStream(TOperationId opId, const T .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderOperation(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_index.cpp index 9df6b0df40..9a1820d8da 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_index.cpp @@ -140,7 +140,8 @@ public: .NotDeleted() .NotUnderDeleting() .IsCommonSensePath() - .IsTable(); + .IsTable() + .NotAsyncReplicaTable(); if (!checks) { result->SetError(checks.GetStatus(), checks.GetError()); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp index c4342bf53e..2ca31c32c5 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp @@ -489,6 +489,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .NotUnderOperation(); if (!context.IsAllowedPrivateTables) { @@ -527,17 +528,19 @@ public: return result; } - if (path.Base()->GetAliveChildren() && alter.HasTTLSettings()) { + bool isReplicated = false; + if (path.Base()->GetAliveChildren()) { for (const auto& [_, childPathId] : path.Base()->GetChildren()) { Y_VERIFY(context.SS->PathsById.contains(childPathId)); auto childPath = context.SS->PathsById.at(childPathId); - if (!childPath->IsTableIndex() || childPath->Dropped()) { + if (!childPath->IsCdcStream() || childPath->Dropped()) { continue; } - Y_VERIFY(context.SS->Indexes.contains(childPathId)); - auto indexInfo = context.SS->Indexes.at(childPathId); + if (isReplicated = childPath->AsyncReplication.IsDefined()) { + break; + } } } @@ -555,6 +558,19 @@ public: return result; } + if (isReplicated) { + for (const auto& [id, column] : alterData->Columns) { + if (column.CreateVersion == alterData->AlterVersion) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, "Cannot add columns to replicated table"); + return result; + } + if (column.DeleteVersion == alterData->AlterVersion) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, "Cannot drop columns of replicated table"); + return result; + } + } + } + TBindingsRoomsChanges bindingChanges; if (context.SS->IsStorageConfigLogic(table)) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h index 269c840090..c7fadcec47 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h @@ -605,6 +605,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .NotUnderOperation() .IsCommonSensePath() //forbid alter impl index tables .NotChildren(); //forbid backup table with indexes diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index b5c93c9418..cf474204e2 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -446,6 +446,9 @@ public: schema.ClearTTLSettings(); } + // replication config is not copied + schema.ClearReplicationConfig(); + NKikimrSchemeOp::TPartitionConfig compilationPartitionConfig; if (!TPartitionConfigMerger::ApplyChanges(compilationPartitionConfig, srcTableInfo->PartitionConfig(), schema.GetPartitionConfig(), AppData(), errStr) || !TPartitionConfigMerger::VerifyCreateParams(compilationPartitionConfig, AppData(), IsShadowDataAllowed(), errStr)) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp index de9d29af56..94eb404bad 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp @@ -128,6 +128,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderDeleting(); @@ -173,11 +174,17 @@ public: switch (streamDesc.GetMode()) { case NKikimrSchemeOp::ECdcStreamModeKeysOnly: - case NKikimrSchemeOp::ECdcStreamModeUpdate: case NKikimrSchemeOp::ECdcStreamModeNewImage: case NKikimrSchemeOp::ECdcStreamModeOldImage: case NKikimrSchemeOp::ECdcStreamModeNewAndOldImages: break; + case NKikimrSchemeOp::ECdcStreamModeUpdate: + if (streamDesc.GetFormat() == NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson) { + result->SetError(NKikimrScheme::StatusInvalidParameter, + "DYNAMODB_STREAMS_JSON format incompatible with specified stream mode"); + return result; + } + break; default: result->SetError(NKikimrScheme::StatusInvalidParameter, TStringBuilder() << "Invalid stream mode: " << static_cast<ui32>(streamDesc.GetMode())); @@ -187,6 +194,23 @@ public: switch (streamDesc.GetFormat()) { case NKikimrSchemeOp::ECdcStreamFormatProto: case NKikimrSchemeOp::ECdcStreamFormatJson: + if (!streamDesc.GetAwsRegion().empty()) { + result->SetError(NKikimrScheme::StatusInvalidParameter, + "AWS_REGION option incompatible with specified stream format"); + return result; + } + break; + case NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson: + if (!AppData()->FeatureFlags.GetEnableChangefeedDynamoDBStreamsFormat()) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, + "DYNAMODB_STREAMS_JSON format is not supported yet"); + return result; + } + if (tablePath.Base()->DocumentApiVersion < 1) { + result->SetError(NKikimrScheme::StatusInvalidParameter, + "DYNAMODB_STREAMS_JSON format incompatible with non-document table"); + return result; + } break; default: result->SetError(NKikimrScheme::StatusInvalidParameter, TStringBuilder() @@ -200,6 +224,14 @@ public: return result; } + TUserAttributes::TPtr userAttrs = new TUserAttributes(1); + if (!userAttrs->ApplyPatch(EUserAttributesOp::CreateChangefeed, streamDesc.GetUserAttributes(), errStr) || + !userAttrs->CheckLimits(errStr)) + { + result->SetError(NKikimrScheme::StatusInvalidParameter, errStr); + return result; + } + auto stream = TCdcStreamInfo::Create(streamDesc); Y_VERIFY(stream); @@ -213,6 +245,7 @@ public: context.DbChanges.PersistPath(pathId); context.DbChanges.PersistPath(tablePath.Base()->PathId); + context.DbChanges.PersistApplyUserAttrs(pathId); context.DbChanges.PersistAlterCdcStream(pathId); context.DbChanges.PersistTxState(OperationId); @@ -227,6 +260,7 @@ public: streamPath.Base()->CreateTxId = OperationId.GetTxId(); streamPath.Base()->LastTxId = OperationId.GetTxId(); streamPath.Base()->PathType = TPathElement::EPathType::EPathTypeCdcStream; + streamPath.Base()->UserAttrs->AlterData = userAttrs; context.SS->CdcStreams[pathId] = stream; context.SS->IncrementPathDbRefCount(pathId); @@ -467,6 +501,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderDeleting(); @@ -598,6 +633,7 @@ TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId opId, const TTx .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderDeleting() .NotUnderOperation(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp index 74a280337d..d72862cef0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp @@ -138,7 +138,8 @@ public: .NotDeleted() .NotUnderDeleting() .IsCommonSensePath() - .IsTable(); + .IsTable() + .NotAsyncReplicaTable(); if (tableIndexCreation.GetState() == NKikimrSchemeOp::EIndexState::EIndexStateReady) { checks diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index 5413fb48a9..ecd9c151ae 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -143,6 +143,7 @@ public: .NotUnderDeleting() .NotUnderOperation() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath(); if (!checks) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_deallocate_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_deallocate_pq.cpp index 9a4fcc5578..011b75af91 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_deallocate_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_deallocate_pq.cpp @@ -119,6 +119,7 @@ public: domainInfo->DecPQReservedStorage(reserve.Storage); domainInfo->AggrDiskSpaceUsage({}, pqGroup->Stats); + context.SS->ChangeDiskSpaceTopicsTotalBytes(domainInfo->GetPQAccountStorage()); context.SS->TabletCounters->Simple()[COUNTER_STREAM_RESERVED_THROUGHPUT].Sub(reserve.Throughput); context.SS->TabletCounters->Simple()[COUNTER_STREAM_RESERVED_STORAGE].Sub(reserve.Storage); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp index 7884c523dc..1ee63f1141 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp @@ -145,6 +145,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .IsUnderOperation() .IsUnderTheSameOperation(OperationId.GetTxId()); @@ -326,6 +327,7 @@ public: .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderDeleting() .NotUnderOperation(); @@ -470,6 +472,7 @@ TVector<ISubOperationBase::TPtr> CreateDropCdcStream(TOperationId opId, const TT .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .NotUnderDeleting() .NotUnderOperation(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp index 2331e78951..ae1901be0f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp @@ -282,6 +282,7 @@ public: .NotDeleted() .NotUnderDeleting() .IsTable() + .NotAsyncReplicaTable() .NotUnderOperation() .IsCommonSensePath(); @@ -416,6 +417,7 @@ TVector<ISubOperationBase::TPtr> CreateDropIndex(TOperationId nextId, const TTxT .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .NotUnderDeleting() .NotUnderOperation() .IsCommonSensePath(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp index b7127cc228..cb57c4c860 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_pq.cpp @@ -224,6 +224,7 @@ public: context.OnComplete.PublishToSchemeBoard(OperationId, subDomainId); } + context.SS->ChangeDiskSpaceTopicsTotalBytes(domainInfo->GetPQAccountStorage()); context.SS->TabletCounters->Simple()[COUNTER_STREAM_RESERVED_THROUGHPUT].Sub(reserve.Throughput); context.SS->TabletCounters->Simple()[COUNTER_STREAM_RESERVED_STORAGE].Sub(reserve.Storage); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp index f90437e97e..3661f89f0f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp @@ -380,6 +380,7 @@ public: .NotDeleted() .NotUnderDeleting() .IsTable() + .NotAsyncReplicaTable() .NotUnderOperation() .IsCommonSensePath(); @@ -489,6 +490,7 @@ TVector<ISubOperationBase::TPtr> CreateConsistentMoveIndex(TOperationId nextId, .IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .NotUnderDeleting() .NotUnderOperation() .IsCommonSensePath(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp index 245e48e54c..0f3dca4f1d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp @@ -581,6 +581,7 @@ public: .NotDeleted() .IsTable() .NotBackupTable() + .NotAsyncReplicaTable() .NotUnderTheSameOperation(OperationId.GetTxId()) .NotUnderOperation(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_table_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_table_index.cpp index 761d047dd3..cb7fb4d88a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_table_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_table_index.cpp @@ -385,6 +385,7 @@ public: .NotUnderDeleting() .IsCommonSensePath() .IsTable() + .NotAsyncReplicaTable() .IsUnderTheSameOperation(OperationId.GetTxId()); if (!checks) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp index 894b65d839..00d587d293 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp @@ -33,6 +33,7 @@ TVector<ISubOperationBase::TPtr> CreateConsistentMoveTable(TOperationId nextId, checks.IsResolved() .NotDeleted() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath(); if (!checks) { diff --git a/ydb/core/tx/schemeshard/schemeshard__pq_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__pq_stats.cpp index 93b2f4cd8e..9e5bd3baae 100644 --- a/ydb/core/tx/schemeshard/schemeshard__pq_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__pq_stats.cpp @@ -65,6 +65,7 @@ bool TTxStoreTopicStats::PersistSingleStats(const TPathId& pathId, const TStatsQ NIceDb::TNiceDb db(txc.DB); Self->PersistPersQueueGroupStats(db, pathId, newStats); + Self->ChangeDiskSpaceTopicsTotalBytes(subDomainInfo->GetPQAccountStorage()); if (subDomainInfo->CheckDiskSpaceQuotas(Self)) { auto subDomainId = Self->ResolvePathIdForDomain(pathId); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp index 55971d9a5c..e6a33ac9ee 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp @@ -88,6 +88,7 @@ public: .NotDeleted() .NotUnderDeleting() .IsTable() + .NotAsyncReplicaTable() .IsCommonSensePath() .IsTheSameDomain(domainPath); diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index a271c24a27..210c80b84b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -72,6 +72,14 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase { if (!checks) { return Reply(std::move(response), Ydb::StatusIds::BAD_REQUEST, checks.GetError()); } + + if (!request.HasUserSID() || !Self->SystemBackupSIDs.contains(request.GetUserSID())) { + checks.ExportsLimit(); + } + + if (!checks) { + return Reply(std::move(response), Ydb::StatusIds::PRECONDITION_FAILED, checks.GetError()); + } } TExportInfo::TPtr exportInfo = nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 6616af5bf9..d2074340a7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -1570,6 +1570,7 @@ void TSchemeShard::PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId) NIceDb::TUpdate<Schema::CdcStream::Mode>(alterData->Mode), NIceDb::TUpdate<Schema::CdcStream::Format>(alterData->Format), NIceDb::TUpdate<Schema::CdcStream::VirtualTimestamps>(alterData->VirtualTimestamps), + NIceDb::TUpdate<Schema::CdcStream::AwsRegion>(alterData->AwsRegion), NIceDb::TUpdate<Schema::CdcStream::State>(alterData->State) ); @@ -1594,6 +1595,7 @@ void TSchemeShard::PersistCdcStreamAlterData(NIceDb::TNiceDb& db, const TPathId& NIceDb::TUpdate<Schema::CdcStreamAlterData::Mode>(alterData->Mode), NIceDb::TUpdate<Schema::CdcStreamAlterData::Format>(alterData->Format), NIceDb::TUpdate<Schema::CdcStreamAlterData::VirtualTimestamps>(alterData->VirtualTimestamps), + NIceDb::TUpdate<Schema::CdcStreamAlterData::AwsRegion>(alterData->AwsRegion), NIceDb::TUpdate<Schema::CdcStreamAlterData::State>(alterData->State) ); } @@ -2385,6 +2387,11 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId Y_PROTOBUF_SUPPRESS_NODISCARD tableInfo->TTLSettings().SerializeToString(&ttlSettings); } + TString replicationConfig; + if (tableInfo->HasReplicationConfig()) { + Y_PROTOBUF_SUPPRESS_NODISCARD tableInfo->ReplicationConfig().SerializeToString(&replicationConfig); + } + if (pathId.OwnerId == TabletID()) { db.Table<Schema::Tables>().Key(pathId.LocalPathId).Update( NIceDb::TUpdate<Schema::Tables::NextColId>(tableInfo->NextColumnId), @@ -2393,7 +2400,8 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId NIceDb::TUpdate<Schema::Tables::AlterTable>(TString()), NIceDb::TUpdate<Schema::Tables::AlterTableFull>(TString()), NIceDb::TUpdate<Schema::Tables::TTLSettings>(ttlSettings), - NIceDb::TUpdate<Schema::Tables::IsBackup>(tableInfo->IsBackup)); + NIceDb::TUpdate<Schema::Tables::IsBackup>(tableInfo->IsBackup), + NIceDb::TUpdate<Schema::Tables::ReplicationConfig>(replicationConfig)); } else { db.Table<Schema::MigratedTables>().Key(pathId.OwnerId, pathId.LocalPathId).Update( NIceDb::TUpdate<Schema::MigratedTables::NextColId>(tableInfo->NextColumnId), @@ -2402,7 +2410,8 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId NIceDb::TUpdate<Schema::MigratedTables::AlterTable>(TString()), NIceDb::TUpdate<Schema::MigratedTables::AlterTableFull>(TString()), NIceDb::TUpdate<Schema::MigratedTables::TTLSettings>(ttlSettings), - NIceDb::TUpdate<Schema::MigratedTables::IsBackup>(tableInfo->IsBackup)); + NIceDb::TUpdate<Schema::MigratedTables::IsBackup>(tableInfo->IsBackup), + NIceDb::TUpdate<Schema::MigratedTables::ReplicationConfig>(replicationConfig)); } for (auto col : tableInfo->Columns) { @@ -6049,6 +6058,10 @@ void TSchemeShard::FillTableDescriptionForShardIdx( tableDescr->SetIsBackup(true); } + if (tinfo->HasReplicationConfig()) { + tableDescr->MutableReplicationConfig()->CopyFrom(tinfo->ReplicationConfig()); + } + // Fill indexes & cdc streams (if any) for (const auto& child : pinfo->GetChildren()) { const auto& childName = child.first; @@ -6534,6 +6547,10 @@ void TSchemeShard::ChangeDiskSpaceTablesTotalBytes(i64 delta) { TabletCounters->Simple()[COUNTER_DISK_SPACE_TABLES_TOTAL_BYTES].Add(delta); } +void TSchemeShard::ChangeDiskSpaceTopicsTotalBytes(ui64 value) { + TabletCounters->Simple()[COUNTER_DISK_SPACE_TOPICS_TOTAL_BYTES].Set(value); +} + void TSchemeShard::ChangeDiskSpaceQuotaExceeded(i64 delta) { TabletCounters->Simple()[COUNTER_DISK_SPACE_QUOTA_EXCEEDED].Add(delta); } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 84d69b7927..5299fcb42b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1223,6 +1223,7 @@ public: void ChangeDiskSpaceTablesDataBytes(i64 delta) override; void ChangeDiskSpaceTablesIndexBytes(i64 delta) override; void ChangeDiskSpaceTablesTotalBytes(i64 delta) override; + void ChangeDiskSpaceTopicsTotalBytes(ui64 value) override; void ChangeDiskSpaceQuotaExceeded(i64 delta) override; void ChangeDiskSpaceHardQuotaBytes(i64 delta) override; void ChangeDiskSpaceSoftQuotaBytes(i64 delta) override; diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 4ef3a16946..a3a879f1d3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -74,6 +74,14 @@ struct TSchemeShard::TImport::TTxCreate: public TSchemeShard::TXxport::TTxBase { if (!checks) { return Reply(std::move(response), Ydb::StatusIds::BAD_REQUEST, checks.GetError()); } + + if (!request.HasUserSID() || !Self->SystemBackupSIDs.contains(request.GetUserSID())) { + checks.ImportsLimit(); + } + + if (!checks) { + return Reply(std::move(response), Ydb::StatusIds::PRECONDITION_FAILED, checks.GetError()); + } } TImportInfo::TPtr importInfo = nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 72ece68992..2cca0726ba 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -225,6 +225,26 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( alterData->TableDescriptionFull->MutableTTLSettings()->CopyFrom(ttl); } + if (op.HasReplicationConfig()) { + const auto& cfg = op.GetReplicationConfig(); + + if (source) { + errStr = "Cannot alter replication config"; + return nullptr; + } + + switch (cfg.GetMode()) { + case NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_NONE: + case NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_READ_ONLY: + break; + default: + errStr = "Unknown replication mode"; + return nullptr; + } + + alterData->TableDescriptionFull->MutableReplicationConfig()->CopyFrom(cfg); + } + alterData->IsBackup = op.GetIsBackup(); if (source && op.KeyColumnNamesSize() == 0) @@ -1217,6 +1237,11 @@ void TTableInfo::FinishAlter() { MutableTTLSettings().Swap(AlterData->TableDescriptionFull->MutableTTLSettings()); } + // Apply replication config + if (AlterData->TableDescriptionFull.Defined() && AlterData->TableDescriptionFull->HasReplicationConfig()) { + MutableReplicationConfig().Swap(AlterData->TableDescriptionFull->MutableReplicationConfig()); + } + // Force FillDescription to regenerate TableDescription ResetDescriptionCache(); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index ba0c811cbb..67233a724d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -414,6 +414,19 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { const NKikimrSchemeOp::TPartitionConfig& PartitionConfig() const { return TableDescription.GetPartitionConfig(); } NKikimrSchemeOp::TPartitionConfig& MutablePartitionConfig() { return *TableDescription.MutablePartitionConfig(); } + bool HasReplicationConfig() { return TableDescription.HasReplicationConfig(); } + const NKikimrSchemeOp::TTableReplicationConfig& ReplicationConfig() { return TableDescription.GetReplicationConfig(); } + NKikimrSchemeOp::TTableReplicationConfig& MutableReplicationConfig() { return *TableDescription.MutableReplicationConfig(); } + + bool IsAsyncReplica() const { + switch (TableDescription.GetReplicationConfig().GetMode()) { + case NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_NONE: + return false; + default: + return true; + } + } + bool HasTTLSettings() const { return TableDescription.HasTTLSettings(); } const NKikimrSchemeOp::TTTLSettings& TTLSettings() const { return TableDescription.GetTTLSettings(); } bool IsTTLEnabled() const { return HasTTLSettings() && TTLSettings().HasEnabled(); } @@ -1323,6 +1336,7 @@ struct IQuotaCounters { virtual void ChangeDiskSpaceTablesDataBytes(i64 delta) = 0; virtual void ChangeDiskSpaceTablesIndexBytes(i64 delta) = 0; virtual void ChangeDiskSpaceTablesTotalBytes(i64 delta) = 0; + virtual void ChangeDiskSpaceTopicsTotalBytes(ui64 value) = 0; virtual void ChangeDiskSpaceQuotaExceeded(i64 delta) = 0; virtual void ChangeDiskSpaceHardQuotaBytes(i64 delta) = 0; virtual void ChangeDiskSpaceSoftQuotaBytes(i64 delta) = 0; @@ -2374,11 +2388,12 @@ struct TCdcStreamInfo : public TSimpleRefCount<TCdcStreamInfo> { static constexpr ui32 MaxInProgressShards = 10; - TCdcStreamInfo(ui64 version, EMode mode, EFormat format, bool vt, EState state) + TCdcStreamInfo(ui64 version, EMode mode, EFormat format, bool vt, const TString& awsRegion, EState state) : AlterVersion(version) , Mode(mode) , Format(format) , VirtualTimestamps(vt) + , AwsRegion(awsRegion) , State(state) {} @@ -2392,12 +2407,12 @@ struct TCdcStreamInfo : public TSimpleRefCount<TCdcStreamInfo> { return result; } - static TPtr New(EMode mode, EFormat format, bool vt) { - return new TCdcStreamInfo(0, mode, format, vt, EState::ECdcStreamStateInvalid); + static TPtr New(EMode mode, EFormat format, bool vt, const TString& awsRegion) { + return new TCdcStreamInfo(0, mode, format, vt, awsRegion, EState::ECdcStreamStateInvalid); } static TPtr Create(const NKikimrSchemeOp::TCdcStreamDescription& desc) { - TPtr result = New(desc.GetMode(), desc.GetFormat(), desc.GetVirtualTimestamps()); + TPtr result = New(desc.GetMode(), desc.GetFormat(), desc.GetVirtualTimestamps(), desc.GetAwsRegion()); TPtr alterData = result->CreateNextVersion(); alterData->State = EState::ECdcStreamStateReady; if (desc.HasState()) { @@ -2411,6 +2426,7 @@ struct TCdcStreamInfo : public TSimpleRefCount<TCdcStreamInfo> { EMode Mode; EFormat Format; bool VirtualTimestamps; + TString AwsRegion; EState State; TCdcStreamInfo::TPtr AlterData = nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 63f5ecde3c..fe020b00db 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -366,6 +366,23 @@ const TPath::TChecker& TPath::TChecker::NotBackupTable(EStatus status) const { << " (" << BasicPathInfo(Path.Base()) << ")"); } +const TPath::TChecker& TPath::TChecker::NotAsyncReplicaTable(EStatus status) const { + if (Failed) { + return *this; + } + + if (!Path.Base()->IsTable()) { + return *this; + } + + if (!Path.IsAsyncReplicaTable()) { + return *this; + } + + return Fail(status, TStringBuilder() << "path is an async replica table" + << " (" << BasicPathInfo(Path.Base()) << ")"); +} + const TPath::TChecker& TPath::TChecker::IsBlockStoreVolume(EStatus status) const { if (Failed) { return *this; @@ -776,6 +793,38 @@ const TPath::TChecker& TPath::TChecker::PathShardsLimit(ui64 delta, EStatus stat << ", delta: " << delta); } +const TPath::TChecker& TPath::TChecker::ExportsLimit(ui64 delta, EStatus status) const { + if (Failed) { + return *this; + } + + TSubDomainInfo::TPtr domainInfo = Path.DomainInfo(); + if (Path.SS->Exports.size() + delta <= domainInfo->GetSchemeLimits().MaxExports) { + return *this; + } + + return Fail(status, TStringBuilder() << "exports count limit exceeded" + << ", limit: " << domainInfo->GetSchemeLimits().MaxExports + << ", exports: " << Path.SS->Exports.size() + << ", delta: " << delta); +} + +const TPath::TChecker& TPath::TChecker::ImportsLimit(ui64 delta, EStatus status) const { + if (Failed) { + return *this; + } + + TSubDomainInfo::TPtr domainInfo = Path.DomainInfo(); + if (Path.SS->Imports.size() + delta <= domainInfo->GetSchemeLimits().MaxImports) { + return *this; + } + + return Fail(status, TStringBuilder() << "imports count limit exceeded" + << ", limit: " << domainInfo->GetSchemeLimits().MaxImports + << ", exports: " << Path.SS->Imports.size() + << ", delta: " << delta); +} + const TPath::TChecker& TPath::TChecker::NotChildren(EStatus status) const { if (Failed) { return *this; @@ -1390,6 +1439,18 @@ bool TPath::IsBackupTable() const { return tableInfo->IsBackup; } +bool TPath::IsAsyncReplicaTable() const { + Y_VERIFY(IsResolved()); + + if (!Base()->IsTable() || !SS->Tables.contains(Base()->PathId)) { + return false; + } + + TTableInfo::TCPtr tableInfo = SS->Tables.at(Base()->PathId); + + return tableInfo->IsAsyncReplica(); +} + bool TPath::IsCdcStream() const { Y_VERIFY(IsResolved()); diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index fa328fe1bd..de59867612 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -60,6 +60,7 @@ public: const TChecker& IsInsideCdcStreamPath(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsTable(EStatus status = EStatus::StatusNameConflict) const; const TChecker& NotBackupTable(EStatus status = EStatus::StatusSchemeError) const; + const TChecker& NotAsyncReplicaTable(EStatus status = EStatus::StatusSchemeError) const; const TChecker& IsBlockStoreVolume(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsFileStore(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsKesus(EStatus status = EStatus::StatusNameConflict) const; @@ -85,6 +86,8 @@ public: const TChecker& IsValidACL(const TString& acl, EStatus status = EStatus::StatusInvalidParameter) const; const TChecker& PQPartitionsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& PQReservedStorageLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; + const TChecker& ExportsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; + const TChecker& ImportsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; }; public: @@ -147,6 +150,7 @@ public: bool IsInsideCdcStreamPath() const; bool IsTableIndex() const; bool IsBackupTable() const; + bool IsAsyncReplicaTable() const; bool IsCdcStream() const; bool IsSequence() const; bool IsReplication() const; diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 39132e276e..7f12a3af94 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -1034,6 +1034,10 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme entry->MutableTTLSettings()->CopyFrom(tableInfo->TTLSettings()); } + if (tableInfo->HasReplicationConfig()) { + entry->MutableReplicationConfig()->CopyFrom(tableInfo->ReplicationConfig()); + } + entry->SetIsBackup(tableInfo->IsBackup); } @@ -1101,9 +1105,19 @@ void TSchemeShard::DescribeCdcStream(const TPathId& pathId, const TString& name, desc.SetMode(info->Mode); desc.SetFormat(info->Format); desc.SetVirtualTimestamps(info->VirtualTimestamps); + desc.SetAwsRegion(info->AwsRegion); PathIdFromPathId(pathId, desc.MutablePathId()); desc.SetState(info->State); desc.SetSchemaVersion(info->AlterVersion); + + Y_VERIFY(PathsById.contains(pathId)); + auto path = PathsById.at(pathId); + + for (const auto& [key, value] : path->UserAttrs->Attrs) { + auto& attr = *desc.AddUserAttributes(); + attr.SetKey(key); + attr.SetValue(value); + } } void TSchemeShard::DescribeSequence(const TPathId& pathId, const TString& name, diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp index a317287412..8f51cf36e8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp @@ -271,6 +271,9 @@ void TPathElement::ApplySpecialAttributes() { case EAttribute::DOCUMENT_API_VERSION: HandleAttributeValue(item.second, DocumentApiVersion); break; + case EAttribute::ASYNC_REPLICATION: + HandleAttributeValue(item.second, AsyncReplication); + break; default: break; } @@ -288,6 +291,13 @@ void TPathElement::HandleAttributeValue(const TString& value, ui64& target) { } } +void TPathElement::HandleAttributeValue(const TString& value, NJson::TJsonValue& target) { + NJson::TJsonValue parsed; + if (NJson::ReadJsonTree(value, &parsed)) { + target = std::move(parsed); + } +} + void TPathElement::ChangeVolumeSpaceBegin(TVolumeSpace newSpace, TVolumeSpace oldSpace) { auto update = [](TVolumeSpaceLimits& limits, ui64 newValue, ui64 oldValue) { if (newValue > oldValue) { diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h index f18bb9eca6..bef48913e8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h @@ -5,14 +5,15 @@ #include "schemeshard_user_attr_limits.h" #include <ydb/core/protos/flat_scheme_op.pb.h> +#include <ydb/core/util/yverify_stream.h> #include <ydb/library/aclib/aclib.h> +#include <library/cpp/json/json_reader.h> + #include <util/generic/map.h> #include <util/generic/ptr.h> #include <util/string/cast.h> -#include <ydb/core/util/yverify_stream.h> - namespace NKikimr { namespace NSchemeShard { @@ -26,6 +27,7 @@ constexpr TStringBuf ATTR_VOLUME_SPACE_LIMIT_SSD_NONREPL = "__volume_space_limit constexpr TStringBuf ATTR_VOLUME_SPACE_LIMIT_SSD_SYSTEM = "__volume_space_limit_ssd_system"; constexpr TStringBuf ATTR_EXTRA_PATH_SYMBOLS_ALLOWED = "__extra_path_symbols_allowed"; constexpr TStringBuf ATTR_DOCUMENT_API_VERSION = "__document_api_version"; +constexpr TStringBuf ATTR_ASYNC_REPLICATION = "__async_replication"; inline bool WeakCheck(char c) { // 33: ! " # $ % & ' ( ) * + , - . / @@ -58,6 +60,7 @@ enum class EAttribute { VOLUME_SPACE_LIMIT_SSD_NONREPL, DOCUMENT_API_VERSION, VOLUME_SPACE_LIMIT_SSD_SYSTEM, + ASYNC_REPLICATION, }; struct TVolumeSpace { @@ -81,6 +84,7 @@ enum class EUserAttributesOp { CreateSubDomain, CreateExtSubDomain, SyncUpdateTenants, + CreateChangefeed, }; struct TUserAttributes: TSimpleRefCount<TUserAttributes> { @@ -116,6 +120,7 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { HANDLE_ATTR(VOLUME_SPACE_LIMIT_SSD_SYSTEM); HANDLE_ATTR(EXTRA_PATH_SYMBOLS_ALLOWED); HANDLE_ATTR(DOCUMENT_API_VERSION); + HANDLE_ATTR(ASYNC_REPLICATION); #undef HANDLE_ATTR return EAttribute::UNKNOWN; } @@ -218,6 +223,12 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { return false; } return CheckAttributeUint64(name, value, errStr, /* minValue = */ 1); + case EAttribute::ASYNC_REPLICATION: + if (op != EUserAttributesOp::CreateChangefeed) { + errStr = Sprintf("UserAttributes: attribute '%s' can only be set during CreateChangefeed", name.c_str()); + return false; + } + return CheckAttributeJson(name, value, errStr); } Y_UNREACHABLE(); @@ -248,6 +259,12 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { return false; } return true; + case EAttribute::ASYNC_REPLICATION: + if (op != EUserAttributesOp::CreateChangefeed) { + errStr = Sprintf("UserAttributes: attribute '%s' can only be set during CreateChangefeed", name.c_str()); + return false; + } + return true; } Y_UNREACHABLE(); @@ -267,7 +284,7 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { if (!TryFromString(value, parsed)) { errStr = Sprintf("UserAttributes: attribute '%s' has invalid value '%s'", name.c_str(), value.c_str()); - return false; + return false; } if (parsed < minValue) { errStr = Sprintf("UserAttributes: attribute '%s' has invalid value '%s' < %" PRIu64, @@ -288,6 +305,16 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> { errStr = Sprintf("UserAttributes::CheckLimits: unsupported attribute '%s'", item.first.c_str()); return true; } + + static bool CheckAttributeJson(const TString& name, const TString& value, TString& errStr) { + NJson::TJsonValue unused; + if (!NJson::ReadJsonTree(value, &unused)) { + errStr = Sprintf("UserAttributes: attribute '%s' has invalid value '%s'", + name.c_str(), value.c_str()); + return false; + } + return true; + } }; struct TPathElement : TSimpleRefCount<TPathElement> { @@ -334,6 +361,7 @@ struct TPathElement : TSimpleRefCount<TPathElement> { TVolumeSpaceLimits VolumeSpaceSSDNonrepl; TVolumeSpaceLimits VolumeSpaceSSDSystem; ui64 DocumentApiVersion = 0; + NJson::TJsonValue AsyncReplication; // Number of references to this path element in the database size_t DbRefCount = 0; @@ -397,6 +425,7 @@ public: void ApplySpecialAttributes(); void HandleAttributeValue(const TString& value, TString& target); void HandleAttributeValue(const TString& value, ui64& target); + void HandleAttributeValue(const TString& value, NJson::TJsonValue& target); void ChangeVolumeSpaceBegin(TVolumeSpace newSpace, TVolumeSpace oldSpace); void ChangeVolumeSpaceCommit(TVolumeSpace newSpace, TVolumeSpace oldSpace); bool CheckVolumeSpaceChange(TVolumeSpace newSpace, TVolumeSpace oldSpace, TString& errStr); diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 49a7ff2425..55e861e8d0 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -117,6 +117,7 @@ struct Schema : NIceDb::Schema { struct PartitioningVersion : Column<7, NScheme::NTypeIds::Uint64> {}; struct TTLSettings : Column<8, NScheme::NTypeIds::String> {}; struct IsBackup : Column<9, NScheme::NTypeIds::Bool> {}; + struct ReplicationConfig : Column<10, NScheme::NTypeIds::String> {}; using TKey = TableKey<TabId>; using TColumns = TableColumns< @@ -128,7 +129,8 @@ struct Schema : NIceDb::Schema { AlterTableFull, PartitioningVersion, TTLSettings, - IsBackup + IsBackup, + ReplicationConfig >; }; @@ -144,6 +146,7 @@ struct Schema : NIceDb::Schema { struct PartitioningVersion : Column<8, NScheme::NTypeIds::Uint64> {}; struct TTLSettings : Column<9, NScheme::NTypeIds::String> {}; struct IsBackup : Column<10, NScheme::NTypeIds::Bool> {}; + struct ReplicationConfig : Column<11, NScheme::NTypeIds::String> {}; using TKey = TableKey<OwnerPathId, LocalPathId>; using TColumns = TableColumns< @@ -156,7 +159,8 @@ struct Schema : NIceDb::Schema { AlterTableFull, PartitioningVersion, TTLSettings, - IsBackup + IsBackup, + ReplicationConfig >; }; @@ -721,6 +725,8 @@ struct Schema : NIceDb::Schema { struct DiskQuotaExceeded : Column<25, NScheme::NTypeIds::Bool> {}; struct SecurityStateVersion : Column<26, NScheme::NTypeIds::Uint64> {}; struct TableCdcStreamsLimit : Column<27, NScheme::NTypeIds::Uint64> {}; + struct ExportsLimit : Column<28, NScheme::NTypeIds::Uint64> {}; + struct ImportsLimit : Column<29, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<PathId>; using TColumns = TableColumns< @@ -750,7 +756,9 @@ struct Schema : NIceDb::Schema { StateVersion, DiskQuotaExceeded, SecurityStateVersion, - TableCdcStreamsLimit + TableCdcStreamsLimit, + ExportsLimit, + ImportsLimit >; }; @@ -1534,9 +1542,10 @@ struct Schema : NIceDb::Schema { struct Mode : Column<5, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamMode; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamModeInvalid; }; struct Format : Column<6, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamFormat; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamFormatInvalid; }; struct VirtualTimestamps : Column<7, NScheme::NTypeIds::Bool> {}; + struct AwsRegion : Column<8, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode, Format, VirtualTimestamps>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode, Format, VirtualTimestamps, AwsRegion>; }; struct CdcStreamAlterData : Table<96> { @@ -1547,9 +1556,10 @@ struct Schema : NIceDb::Schema { struct Mode : Column<5, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamMode; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamModeInvalid; }; struct Format : Column<6, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::ECdcStreamFormat; static constexpr Type Default = NKikimrSchemeOp::ECdcStreamFormatInvalid; }; struct VirtualTimestamps : Column<7, NScheme::NTypeIds::Bool> {}; + struct AwsRegion : Column<8, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode, Format, VirtualTimestamps>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, State, Mode, Format, VirtualTimestamps, AwsRegion>; }; struct CdcStreamScanShardStatus : Table<103> { diff --git a/ydb/core/tx/schemeshard/schemeshard_types.cpp b/ydb/core/tx/schemeshard/schemeshard_types.cpp index cc3c439f5f..443cafd3e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_types.cpp @@ -2,52 +2,62 @@ namespace NKikimr::NSchemeShard { -TSchemeLimits::TSchemeLimits(const NKikimrScheme::TSchemeLimits& proto) { +TSchemeLimits TSchemeLimits::FromProto(const NKikimrScheme::TSchemeLimits& proto) { + TSchemeLimits result; + if (proto.HasMaxDepth()) { - MaxDepth = proto.GetMaxDepth(); + result.MaxDepth = proto.GetMaxDepth(); } if (proto.HasMaxPaths()) { - MaxPaths = proto.GetMaxPaths(); + result.MaxPaths = proto.GetMaxPaths(); } if (proto.HasMaxChildrenInDir()) { - MaxChildrenInDir = proto.GetMaxChildrenInDir(); + result.MaxChildrenInDir = proto.GetMaxChildrenInDir(); } if (proto.HasMaxAclBytesSize()) { - MaxAclBytesSize = proto.GetMaxAclBytesSize(); + result.MaxAclBytesSize = proto.GetMaxAclBytesSize(); } if (proto.HasMaxTableColumns()) { - MaxTableColumns = proto.GetMaxTableColumns(); + result.MaxTableColumns = proto.GetMaxTableColumns(); } if (proto.HasMaxTableColumnNameLength()) { - MaxTableColumnNameLength = proto.GetMaxTableColumnNameLength(); + result.MaxTableColumnNameLength = proto.GetMaxTableColumnNameLength(); } if (proto.HasMaxTableKeyColumns()) { - MaxTableKeyColumns = proto.GetMaxTableKeyColumns(); + result.MaxTableKeyColumns = proto.GetMaxTableKeyColumns(); } if (proto.HasMaxTableIndices()) { - MaxTableIndices = proto.GetMaxTableIndices(); + result.MaxTableIndices = proto.GetMaxTableIndices(); } if (proto.HasMaxTableCdcStreams()) { - MaxTableCdcStreams = proto.GetMaxTableCdcStreams(); + result.MaxTableCdcStreams = proto.GetMaxTableCdcStreams(); } if (proto.HasMaxShards()) { - MaxShards = proto.GetMaxShards(); + result.MaxShards = proto.GetMaxShards(); } if (proto.HasMaxShardsInPath()) { - MaxShardsInPath = proto.GetMaxShardsInPath(); + result.MaxShardsInPath = proto.GetMaxShardsInPath(); } if (proto.HasMaxConsistentCopyTargets()) { - MaxConsistentCopyTargets = proto.GetMaxConsistentCopyTargets(); + result.MaxConsistentCopyTargets = proto.GetMaxConsistentCopyTargets(); } if (proto.HasMaxPathElementLength()) { - MaxPathElementLength = proto.GetMaxPathElementLength(); + result.MaxPathElementLength = proto.GetMaxPathElementLength(); } if (proto.HasExtraPathSymbolsAllowed()) { - ExtraPathSymbolsAllowed = proto.GetExtraPathSymbolsAllowed(); + result.ExtraPathSymbolsAllowed = proto.GetExtraPathSymbolsAllowed(); } if (proto.HasMaxPQPartitions()) { - MaxPQPartitions = proto.GetMaxPQPartitions(); + result.MaxPQPartitions = proto.GetMaxPQPartitions(); + } + if (proto.HasMaxExports()) { + result.MaxExports = proto.GetMaxExports(); + } + if (proto.HasMaxImports()) { + result.MaxImports = proto.GetMaxImports(); } + + return result; } NKikimrScheme::TSchemeLimits TSchemeLimits::AsProto() const { @@ -72,6 +82,9 @@ NKikimrScheme::TSchemeLimits TSchemeLimits::AsProto() const { result.SetMaxPQPartitions(MaxPQPartitions); + result.SetMaxExports(MaxExports); + result.SetMaxImports(MaxImports); + return result; } diff --git a/ydb/core/tx/schemeshard/schemeshard_types.h b/ydb/core/tx/schemeshard/schemeshard_types.h index be2368f7d6..d527b0b77f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_types.h @@ -35,9 +35,11 @@ struct TSchemeLimits { // pq group ui64 MaxPQPartitions = 1000000; - TSchemeLimits() = default; - explicit TSchemeLimits(const NKikimrScheme::TSchemeLimits& proto); + // export & import + ui64 MaxExports = 10; + ui64 MaxImports = 10; + static TSchemeLimits FromProto(const NKikimrScheme::TSchemeLimits& proto); NKikimrScheme::TSchemeLimits AsProto() const; }; diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream.cpp b/ydb/core/tx/schemeshard/ut_cdc_stream.cpp index 619399646d..7946c6cf4d 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/ut_cdc_stream.cpp @@ -4,6 +4,9 @@ #include <ydb/core/tx/schemeshard/schemeshard_impl.h> #include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_writer.h> + +#include <util/string/escape.h> #include <util/string/printf.h> using namespace NKikimr; @@ -140,6 +143,224 @@ Y_UNIT_TEST_SUITE(TCdcStreamTests) { } } + Y_UNIT_TEST(Attributes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + Format: ECdcStreamFormatProto + UserAttributes { Key: "key" Value: "value" } + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + [=](const NKikimrScheme::TEvDescribeSchemeResult& record) { + const auto& table = record.GetPathDescription().GetTable(); + UNIT_ASSERT_VALUES_EQUAL(table.CdcStreamsSize(), 1); + + const auto& stream = table.GetCdcStreams(0); + UNIT_ASSERT_VALUES_EQUAL(stream.UserAttributesSize(), 1); + + const auto& attr = stream.GetUserAttributes(0); + UNIT_ASSERT_VALUES_EQUAL(attr.GetKey(), "key"); + UNIT_ASSERT_VALUES_EQUAL(attr.GetValue(), "value"); + } + }); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), { + NLs::UserAttrsHas({ + {"key", "value"}, + }) + }); + } + + Y_UNIT_TEST(ReplicationAttribute) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + Format: ECdcStreamFormatProto + UserAttributes { Key: "__async_replication" Value: "value" } + } + )", {NKikimrScheme::StatusInvalidParameter}); + + NJson::TJsonValue json; + json["id"] = "some-id"; + json["path"] = "/some/path"; + const auto jsonString = NJson::WriteJson(json, false); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", Sprintf(R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + Format: ECdcStreamFormatProto + UserAttributes { Key: "__async_replication" Value: "%s" } + } + )", EscapeC(jsonString).c_str())); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), { + NLs::UserAttrsHas({ + {"__async_replication", jsonString}, + }) + }); + + // now it is forbidden to change the scheme + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "extra" Type: "Uint64" } + )", {NKikimrScheme::StatusPreconditionFailed}); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + DropColumns { Name: "value" } + )", {NKikimrScheme::StatusPreconditionFailed}); + + // drop stream + TestDropCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamName: "Stream" + )"); + env.TestWaitNotification(runtime, txId); + + // now it is allowed to change the scheme + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "extra" Type: "Uint64" } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + DropColumns { Name: "value" } + )"); + env.TestWaitNotification(runtime, txId); + } + + Y_UNIT_TEST(DocApi) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableChangefeedDynamoDBStreamsFormat(true)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "RowTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "DocumentTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )", {NKikimrScheme::StatusAccepted}, AlterUserAttrs({{"__document_api_version", "1"}})); + env.TestWaitNotification(runtime, txId); + + // non-document table + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "RowTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeNewAndOldImages + Format: ECdcStreamFormatDynamoDBStreamsJson + } + )", {NKikimrScheme::StatusInvalidParameter}); + + // invalid mode + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "DocumentTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeUpdate + Format: ECdcStreamFormatDynamoDBStreamsJson + } + )", {NKikimrScheme::StatusInvalidParameter}); + + // invalid aws region + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "DocumentTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + Format: ECdcStreamFormatProto + AwsRegion: "foo" + } + )", {NKikimrScheme::StatusInvalidParameter}); + + // ok + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "DocumentTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeNewAndOldImages + Format: ECdcStreamFormatDynamoDBStreamsJson + AwsRegion: "ru-central1" + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/DocumentTable/Stream"), { + NLs::PathExist, + NLs::StreamMode(NKikimrSchemeOp::ECdcStreamModeNewAndOldImages), + NLs::StreamFormat(NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson), + NLs::StreamAwsRegion("ru-central1"), + }); + } + + Y_UNIT_TEST(DocApiNegative) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableChangefeedDynamoDBStreamsFormat(false)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "DocumentTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )", {NKikimrScheme::StatusAccepted}, AlterUserAttrs({{"__document_api_version", "1"}})); + env.TestWaitNotification(runtime, txId); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "DocumentTable" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeNewAndOldImages + Format: ECdcStreamFormatDynamoDBStreamsJson + AwsRegion: "ru-central1" + } + )", {NKikimrScheme::StatusPreconditionFailed}); + } + Y_UNIT_TEST(Negative) { TTestBasicRuntime runtime; TTestEnv env(runtime, TTestEnvOptions().EnableProtoSourceIdInfo(true)); @@ -1052,7 +1273,7 @@ Y_UNIT_TEST_SUITE(TCdcStreamWithInitialScanTests) { // the table is locked now TestAlterTable(runtime, ++txId, "/MyRoot", R"( Name: "Table" - Columns { Name: "extra" Type: "Uint64"} + Columns { Name: "extra" Type: "Uint64" } )", {NKikimrScheme::StatusMultipleModifications}); TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( @@ -1074,7 +1295,7 @@ Y_UNIT_TEST_SUITE(TCdcStreamWithInitialScanTests) { // the table is no longer locked TestAlterTable(runtime, ++txId, "/MyRoot", R"( Name: "Table" - Columns { Name: "extra" Type: "Uint64"} + Columns { Name: "extra" Type: "Uint64" } )"); env.TestWaitNotification(runtime, txId); diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp index 7c65fb0537..fb10c2298d 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots.cpp @@ -66,6 +66,45 @@ Y_UNIT_TEST_SUITE(TCdcStreamWithRebootsTests) { CreateStream({}, true); } + Y_UNIT_TEST(CreateStreamWithAwsRegion) { + TTestWithReboots t; + t.GetTestEnvOptions().EnableChangefeedDynamoDBStreamsFormat(true); + + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )", {NKikimrScheme::StatusAccepted}, AlterUserAttrs({{"__document_api_version", "1"}})); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestCreateCdcStream(runtime, ++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeNewAndOldImages + Format: ECdcStreamFormatDynamoDBStreamsJson + AwsRegion: "ru-central1" + } + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), { + NLs::PathExist, + NLs::StreamMode(NKikimrSchemeOp::ECdcStreamModeNewAndOldImages), + NLs::StreamFormat(NKikimrSchemeOp::ECdcStreamFormatDynamoDBStreamsJson), + NLs::StreamAwsRegion("ru-central1"), + }); + } + }); + } + Y_UNIT_TEST(DisableStream) { TTestWithReboots t; t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { @@ -302,6 +341,47 @@ Y_UNIT_TEST_SUITE(TCdcStreamWithRebootsTests) { }); } + Y_UNIT_TEST(Attributes) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + auto request = CreateCdcStreamRequest(++t.TxId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + Format: ECdcStreamFormatProto + UserAttributes { Key: "key" Value: "value" } + } + )"); + t.TestEnv->ReliablePropose(runtime, request, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + NKikimrScheme::StatusMultipleModifications, + }); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Stream"), { + NLs::UserAttrsHas({ + {"key", "value"}, + }) + }); + } + }); + } + Y_UNIT_TEST(RacySplitAndDropTable) { TTestWithReboots t; t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { diff --git a/ydb/core/tx/schemeshard/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export.cpp index c8d97aafc0..907e564aef 100644 --- a/ydb/core/tx/schemeshard/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export.cpp @@ -90,16 +90,26 @@ namespace { } for (const auto& table : tables) { - TestCreateTable(runtime, schemeshardId, ++txId, dbName, table); + TestCreateTable(runtime, schemeshardId, ++txId, dbName, table, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + }); env.TestWaitNotification(runtime, txId, schemeshardId); } runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - TestExport(runtime, schemeshardId, ++txId, dbName, request, userSID); + const auto initialStatus = expectedStatus == Ydb::StatusIds::PRECONDITION_FAILED + ? expectedStatus + : Ydb::StatusIds::SUCCESS; + TestExport(runtime, schemeshardId, ++txId, dbName, request, userSID, initialStatus); env.TestWaitNotification(runtime, txId, schemeshardId); + if (initialStatus != Ydb::StatusIds::SUCCESS) { + return; + } + const ui64 exportId = txId; TestGetExport(runtime, schemeshardId, exportId, dbName, expectedStatus); @@ -1060,4 +1070,42 @@ partitioning_settings { env.TestWaitNotification(runtime, exportId); TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::SUCCESS); } + + Y_UNIT_TEST(ShouldCheckQuotas) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + const TString userSID = "user@builtin"; + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().SystemBackupSIDs({userSID})); + + TSchemeLimits lowLimits; + lowLimits.MaxExports = 0; + SetSchemeshardSchemaLimits(runtime, lowLimits); + + const TVector<TString> tables = { + R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", + }; + const TString request = Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table" + destination_prefix: "" + } + } + )", port); + + Run(runtime, env, tables, request, Ydb::StatusIds::PRECONDITION_FAILED); + Run(runtime, env, tables, request, Ydb::StatusIds::SUCCESS, "/MyRoot", false, userSID); + } } diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 349e636972..6344521fc2 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -1088,29 +1088,34 @@ namespace NSchemeShardUT_Private { return TestForgetExport(runtime, TTestTxConfig::SchemeShard, txId, dbName, exportId, expectedStatus); } - void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr) { + void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { NKikimrImport::TCreateImportRequest request; UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(requestStr, &request)); - ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), new TEvImport::TEvCreateImportRequest(id, dbName, request)); + auto ev = MakeHolder<TEvImport::TEvCreateImportRequest>(id, dbName, request); + if (userSID) { + ev->Record.SetUserSID(userSID); + } + + ForwardToTablet(runtime, schemeshardId, runtime.AllocateEdgeActor(), ev.Release()); } - void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr) { - AsyncImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr); + void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { + AsyncImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, userSID); } - void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, + void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, Ydb::StatusIds::StatusCode expectedStatus) { - AsyncImport(runtime, schemeshardId, id, dbName, requestStr); + AsyncImport(runtime, schemeshardId, id, dbName, requestStr, userSID); TAutoPtr<IEventHandle> handle; auto ev = runtime.GrabEdgeEvent<TEvImport::TEvCreateImportResponse>(handle); UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetEntry().GetStatus(), expectedStatus); } - void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, + void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, Ydb::StatusIds::StatusCode expectedStatus) { - TestImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, expectedStatus); + TestImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, userSID, expectedStatus); } NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, @@ -1249,29 +1254,6 @@ namespace NSchemeShardUT_Private { return result.GetValue().GetStruct(0).GetOptional().HasOptional(); } - NKikimrProto::EReplyStatus LocalSchemeTx(TTestActorRuntime& runtime, ui64 tabletId, const TString& schemeChangesStr, bool dryRun, - NTabletFlatScheme::TSchemeChanges& scheme, TString& err) { - TActorId sender = runtime.AllocateEdgeActor(); - - auto evTx = new TEvTablet::TEvLocalSchemeTx; - evTx->Record.SetDryRun(dryRun); - auto schemeChanges = evTx->Record.MutableSchemeChanges(); - bool parseResult = ::google::protobuf::TextFormat::ParseFromString(schemeChangesStr, schemeChanges); - UNIT_ASSERT_C(parseResult, "protobuf parsing failed"); - - ForwardToTablet(runtime, tabletId, sender, evTx); - - TAutoPtr<IEventHandle> handle; - auto event = runtime.GrabEdgeEvent<TEvTablet::TEvLocalSchemeTxResponse>(handle); - UNIT_ASSERT(event); - - err = event->Record.GetErrorReason(); - scheme.CopyFrom(event->Record.GetFullScheme()); - - // emulate enum behavior from proto3 - return static_cast<NKikimrProto::EReplyStatus>(event->Record.GetStatus()); - } - ui64 GetDatashardState(TTestActorRuntime& runtime, ui64 tabletId) { NKikimrMiniKQL::TResult result; TString err; @@ -1332,24 +1314,6 @@ namespace NSchemeShardUT_Private { return GetDatashardSysTableValue(runtime, tabletId, 20); } - ui64 GetExecutorCacheSize(TTestActorRuntime& runtime, ui64 tabletId) { - NTabletFlatScheme::TSchemeChanges scheme; - TString err; - NKikimrProto::EReplyStatus status = LocalSchemeTx(runtime, tabletId, "", true, scheme, err); - UNIT_ASSERT_VALUES_EQUAL(status, NKikimrProto::EReplyStatus::OK); - //Cdbg << scheme << "\n"; - // looking for "Delta { DeltaType: UpdateExecutorInfo ExecutorCacheSize: 33554432 }" - for (ui32 i = 0; i < scheme.DeltaSize(); ++i) { - const auto& d = scheme.GetDelta(i); - if (d.GetDeltaType() == NTabletFlatScheme::TAlterRecord::UpdateExecutorInfo) { - return d.GetExecutorCacheSize(); - } - } - UNIT_ASSERT_C(false, "UpdateExecutorInfo delta record not found"); - return -1; - } - - bool GetFastLogPolicy(TTestActorRuntime& runtime, ui64 tabletId) { NTabletFlatScheme::TSchemeChanges scheme; TString err; @@ -1482,14 +1446,17 @@ namespace NSchemeShardUT_Private { (let maxPathLength '('PathElementLength (Uint64 '%lu))) (let extraSymbols '('ExtraPathSymbolsAllowed (Utf8 '"%s"))) (let pqPartitions '('PQPartitionsLimit (Uint64 '%lu))) - (let ret (AsList (UpdateRow 'SubDomains key '(depth paths child acl columns colName keyCols indices streams shards pathShards consCopy maxPathLength extraSymbols pqPartitions)))) + (let exports '('ExportsLimit (Uint64 '%lu))) + (let imports '('ImportsLimit (Uint64 '%lu))) + (let ret (AsList (UpdateRow 'SubDomains key '(depth paths child acl columns colName keyCols indices streams shards pathShards consCopy maxPathLength extraSymbols pqPartitions exports imports)))) (return ret) ) )", domainId, limits.MaxDepth, limits.MaxPaths, limits.MaxChildrenInDir, limits.MaxAclBytesSize, limits.MaxTableColumns, limits.MaxTableColumnNameLength, limits.MaxTableKeyColumns, limits.MaxTableIndices, limits.MaxTableCdcStreams, limits.MaxShards, limits.MaxShardsInPath, limits.MaxConsistentCopyTargets, - limits.MaxPathElementLength, escapedStr.c_str(), limits.MaxPQPartitions); + limits.MaxPathElementLength, escapedStr.c_str(), limits.MaxPQPartitions, + limits.MaxExports, limits.MaxImports); Cdbg << prog << "\n"; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, schemeShard, prog, result, err); Cdbg << result << "\n"; diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index d637c218a6..4801040c7e 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -8,6 +8,7 @@ #include <ydb/core/engine/mkql_engine_flat.h> #include <ydb/core/persqueue/ut/common/pq_ut_common.h> #include <ydb/core/protos/tx_datashard.pb.h> +#include <ydb/core/testlib/tx_helpers.h> #include <ydb/core/testlib/minikql_compile.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> @@ -61,7 +62,6 @@ namespace NSchemeShardUT_Private { ////////// tablet NKikimrProto::EReplyStatus LocalMiniKQL(TTestActorRuntime& runtime, ui64 tabletId, const TString& query, NKikimrMiniKQL::TResult& result, TString& err); NKikimrMiniKQL::TResult LocalMiniKQL(TTestActorRuntime& runtime, ui64 tabletId, const TString& query); - NKikimrProto::EReplyStatus LocalSchemeTx(TTestActorRuntime& runtime, ui64 tabletId, const TString& schemeChangesStr, bool dryRun, NTabletFlatScheme::TSchemeChanges& scheme, TString& err); bool CheckLocalRowExists(TTestActorRuntime& runtime, ui64 tabletId, const TString& tableName, const TString& keyColumn, ui64 keyValue); @@ -362,11 +362,11 @@ namespace NSchemeShardUT_Private { Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); ////////// import - void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr); - void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr); - void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, + void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); + void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); + void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, + void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); @@ -390,7 +390,6 @@ namespace NSchemeShardUT_Private { ui64 GetTxReadSizeLimit(TTestActorRuntime& runtime, ui64 tabletId); ui64 GetStatDisabled(TTestActorRuntime& runtime, ui64 tabletId); - ui64 GetExecutorCacheSize(TTestActorRuntime& runtime, ui64 tabletId); bool GetFastLogPolicy(TTestActorRuntime& runtime, ui64 tabletId); bool GetByKeyFilterEnabled(TTestActorRuntime& runtime, ui64 tabletId, ui32 table); bool GetEraseCacheEnabled(TTestActorRuntime& runtime, ui64 tabletId, ui32 table); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 94f2d5b1b4..c8b26b727d 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -786,6 +786,12 @@ TCheckFunc StreamVirtualTimestamps(bool value) { }; } +TCheckFunc StreamAwsRegion(const TString& value) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetCdcStreamDescription().GetAwsRegion(), value); + }; +} + TCheckFunc RetentionPeriod(const TDuration& value) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(value.Seconds(), record.GetPathDescription().GetPersQueueGroup() diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 4597f90465..7bea73b077 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -131,6 +131,7 @@ namespace NLs { TCheckFunc StreamFormat(NKikimrSchemeOp::ECdcStreamFormat format); TCheckFunc StreamState(NKikimrSchemeOp::ECdcStreamState state); TCheckFunc StreamVirtualTimestamps(bool value); + TCheckFunc StreamAwsRegion(const TString& value); TCheckFunc RetentionPeriod(const TDuration& value); TCheckFunc HasBackupInFly(ui64 txId); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index ef5917450a..f64c3c9e48 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -531,6 +531,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe app.SetEnableNotNullDataColumns(opts.EnableNotNullDataColumns_); app.SetEnableAlterDatabaseCreateHiveFirst(opts.EnableAlterDatabaseCreateHiveFirst_); app.SetEnableTopicDiskSubDomainQuota(opts.EnableTopicDiskSubDomainQuota_); + app.SetEnableChangefeedDynamoDBStreamsFormat(opts.EnableChangefeedDynamoDBStreamsFormat_); if (opts.DisableStatsBatching_.value_or(false)) { app.SchemeShardConfig.SetStatsMaxBatchSize(0); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index 76619af13e..1596c9f353 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -50,6 +50,7 @@ namespace NSchemeShardUT_Private { OPTION(std::optional<bool>, EnableNotNullDataColumns, std::nullopt); OPTION(std::optional<bool>, EnableAlterDatabaseCreateHiveFirst, std::nullopt); OPTION(std::optional<bool>, EnableTopicDiskSubDomainQuota, std::nullopt); + OPTION(std::optional<bool>, EnableChangefeedDynamoDBStreamsFormat, std::nullopt); #undef OPTION }; diff --git a/ydb/core/tx/schemeshard/ut_replication.cpp b/ydb/core/tx/schemeshard/ut_replication.cpp index 0f511f2188..3d2ccc14c3 100644 --- a/ydb/core/tx/schemeshard/ut_replication.cpp +++ b/ydb/core/tx/schemeshard/ut_replication.cpp @@ -93,4 +93,113 @@ Y_UNIT_TEST_SUITE(TReplicationTests) { TestLs(runtime, "/MyRoot/Replication", false, NLs::PathExist); } + void CreateReplicatedTable(NKikimrSchemeOp::TTableReplicationConfig::EReplicationMode mode) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + ReplicationConfig { + Mode: %s + } + )", NKikimrSchemeOp::TTableReplicationConfig::EReplicationMode_Name(mode).c_str())); + env.TestWaitNotification(runtime, txId); + + const auto desc = DescribePath(runtime, "/MyRoot/Table"); + const auto& table = desc.GetPathDescription().GetTable(); + UNIT_ASSERT(table.HasReplicationConfig()); + UNIT_ASSERT_EQUAL(table.GetReplicationConfig().GetMode(), mode); + } + + Y_UNIT_TEST(CreateReplicatedTable) { + CreateReplicatedTable(NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_NONE); + CreateReplicatedTable(NKikimrSchemeOp::TTableReplicationConfig::REPLICATION_MODE_READ_ONLY); + } + + Y_UNIT_TEST(CannotAddReplicationConfig) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + ReplicationConfig { + Mode: REPLICATION_MODE_READ_ONLY + } + )", {NKikimrScheme::StatusInvalidParameter}); + } + + Y_UNIT_TEST(AlterReplicatedTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + ReplicationConfig { + Mode: REPLICATION_MODE_READ_ONLY + } + )"); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + DropColumns { Name: "value" } + )", {NKikimrScheme::StatusSchemeError}); + + TestCreateCdcStream(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + StreamDescription { + Name: "Stream" + Mode: ECdcStreamModeKeysOnly + Format: ECdcStreamFormatProto + } + )", {NKikimrScheme::StatusSchemeError}); + + TestBuildIndex(runtime, ++txId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", "by_value", {"value"}, + Ydb::StatusIds::BAD_REQUEST); + } + + Y_UNIT_TEST(CopyReplicatedTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint64" } + KeyColumnNames: ["key"] + ReplicationConfig { + Mode: REPLICATION_MODE_READ_ONLY + } + )"); + env.TestWaitNotification(runtime, txId); + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "CopyTable" + CopyFromTable: "/MyRoot/Table" + )"); + env.TestWaitNotification(runtime, txId); + + const auto desc = DescribePath(runtime, "/MyRoot/CopyTable"); + const auto& table = desc.GetPathDescription().GetTable(); + UNIT_ASSERT(!table.HasReplicationConfig()); + } + } // TReplicationTests diff --git a/ydb/core/tx/schemeshard/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore.cpp index f7cfe784ed..dbaed89d19 100644 --- a/ydb/core/tx/schemeshard/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore.cpp @@ -1477,8 +1477,8 @@ Y_UNIT_TEST_SUITE(TImportTests) { void Run(TTestBasicRuntime& runtime, TTestEnv& env, THashMap<TString, TString>&& data, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false) { - + const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "") + { ui64 id = 100; TPortManager portManager; @@ -1557,18 +1557,25 @@ Y_UNIT_TEST_SUITE(TImportTests) { runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - TestImport(runtime, schemeshardId, ++id, dbName, Sprintf(request.data(), port)); + const auto initialStatus = expectedStatus == Ydb::StatusIds::PRECONDITION_FAILED + ? expectedStatus + : Ydb::StatusIds::SUCCESS; + TestImport(runtime, schemeshardId, ++id, dbName, Sprintf(request.data(), port), userSID, initialStatus); env.TestWaitNotification(runtime, id, schemeshardId); + if (initialStatus != Ydb::StatusIds::SUCCESS) { + return; + } + TestGetImport(runtime, schemeshardId, id, dbName, expectedStatus); } void Run(TTestBasicRuntime& runtime, THashMap<TString, TString>&& data, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false) { + const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "") { TTestEnv env(runtime, TTestEnvOptions()); - Run(runtime, env, std::move(data), request, expectedStatus, dbName, serverless); + Run(runtime, env, std::move(data), request, expectedStatus, dbName, serverless, userSID); } Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { @@ -2322,6 +2329,42 @@ Y_UNIT_TEST_SUITE(TImportTests) { .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpApplyIndexBuild; }); } + + Y_UNIT_TEST(ShouldCheckQuotas) { + const TString userSID = "user@builtin"; + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().SystemBackupSIDs({userSID})); + + TSchemeLimits lowLimits; + lowLimits.MaxImports = 0; + SetSchemeshardSchemaLimits(runtime, lowLimits); + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}); + + const TString request = R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )"; + + Run(runtime, env, ConvertTestData(data), request, Ydb::StatusIds::PRECONDITION_FAILED); + Run(runtime, env, ConvertTestData(data), request, Ydb::StatusIds::SUCCESS, "/MyRoot", false, userSID); + } } Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { diff --git a/ydb/core/tx/schemeshard/ut_subdomain.cpp b/ydb/core/tx/schemeshard/ut_subdomain.cpp index ec752a4d28..2fd2050a03 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain.cpp +++ b/ydb/core/tx/schemeshard/ut_subdomain.cpp @@ -2316,7 +2316,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 3 PartitionPerTablet: 2 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 10 WriteSpeedInBytesPerSecond : 1000}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 10 WriteSpeedInBytesPerSecond : 1000} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )", {NKikimrScheme::StatusResourceExhausted}); env.TestWaitNotification(runtime, txId - 1); @@ -2333,7 +2333,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 1 PartitionPerTablet: 2 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 200001}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 200001} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )", {NKikimrScheme::StatusResourceExhausted}); env.TestWaitNotification(runtime, txId - 1); @@ -2351,7 +2351,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 1 PartitionPerTablet: 1 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )"); env.TestWaitNotification(runtime, txId - 1); @@ -2368,7 +2368,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 2 PartitionPerTablet: 1 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )"); env.TestWaitNotification(runtime, txId - 1); @@ -2385,7 +2385,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 2 PartitionPerTablet: 1 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100001}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100001} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )", {NKikimrScheme::StatusResourceExhausted}); env.TestWaitNotification(runtime, txId - 1); @@ -3027,7 +3027,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 3 PartitionPerTablet: 2 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 10 WriteSpeedInBytesPerSecond : 1000}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 10 WriteSpeedInBytesPerSecond : 1000} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )", {NKikimrScheme::StatusResourceExhausted}); env.TestWaitNotification(runtime, txId - 1); @@ -3042,7 +3042,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 1 PartitionPerTablet: 2 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 200001}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 200001} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )", {NKikimrScheme::StatusResourceExhausted}); env.TestWaitNotification(runtime, txId - 1); @@ -3058,7 +3058,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 1 PartitionPerTablet: 1 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )"); env.TestWaitNotification(runtime, txId - 1); @@ -3073,7 +3073,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 2 PartitionPerTablet: 1 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100000} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )"); env.TestWaitNotification(runtime, txId - 1); @@ -3088,7 +3088,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) { Name: "Isolda" TotalGroupCount: 2 PartitionPerTablet: 1 - PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100001}} + PQTabletConfig: {PartitionConfig { LifetimeSeconds : 1 WriteSpeedInBytesPerSecond : 100001} MeteringMode: METERING_MODE_RESERVED_CAPACITY} )", {NKikimrScheme::StatusResourceExhausted}); env.TestWaitNotification(runtime, txId - 1); diff --git a/ydb/core/tx/tx_proxy/CMakeLists.darwin.txt b/ydb/core/tx/tx_proxy/CMakeLists.darwin.txt index 2229fe2fd9..ad58f71ecb 100644 --- a/ydb/core/tx/tx_proxy/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_proxy/CMakeLists.darwin.txt @@ -26,6 +26,7 @@ target_link_libraries(core-tx-tx_proxy PUBLIC ydb-core-actorlib_impl ydb-core-base core-blobstorage-base + ydb-core-docapi ydb-core-engine ydb-core-formats ydb-core-grpc_services diff --git a/ydb/core/tx/tx_proxy/CMakeLists.linux-aarch64.txt b/ydb/core/tx/tx_proxy/CMakeLists.linux-aarch64.txt index 91190a235f..df8ace19cc 100644 --- a/ydb/core/tx/tx_proxy/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/tx_proxy/CMakeLists.linux-aarch64.txt @@ -27,6 +27,7 @@ target_link_libraries(core-tx-tx_proxy PUBLIC ydb-core-actorlib_impl ydb-core-base core-blobstorage-base + ydb-core-docapi ydb-core-engine ydb-core-formats ydb-core-grpc_services diff --git a/ydb/core/tx/tx_proxy/CMakeLists.linux.txt b/ydb/core/tx/tx_proxy/CMakeLists.linux.txt index 91190a235f..df8ace19cc 100644 --- a/ydb/core/tx/tx_proxy/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_proxy/CMakeLists.linux.txt @@ -27,6 +27,7 @@ target_link_libraries(core-tx-tx_proxy PUBLIC ydb-core-actorlib_impl ydb-core-base core-blobstorage-base + ydb-core-docapi ydb-core-engine ydb-core-formats ydb-core-grpc_services diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp index d5a916ae51..5e40614f4b 100644 --- a/ydb/core/tx/tx_proxy/schemereq.cpp +++ b/ydb/core/tx/tx_proxy/schemereq.cpp @@ -1,5 +1,6 @@ #include "proxy.h" +#include <ydb/core/docapi/traits.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/base/tablet_pipe.h> @@ -16,9 +17,6 @@ namespace NKikimr { namespace NTxProxy { -static constexpr TStringBuf DocApiRequestType = "_document_api_request"; -static constexpr TStringBuf DocApiTableVersionAttribute = "__document_api_version"; - template<typename TDerived> struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { using TBase = TActorBootstrapped<TDerived>; @@ -944,7 +942,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { } static bool IsDocApiRestricted(const NKikimrTxUserProxy::TEvProposeTransaction& tx) { - if (tx.GetRequestType() == DocApiRequestType) { + if (tx.GetRequestType() == NDocApi::RequestType) { return false; } @@ -960,7 +958,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { bool CheckDocApi(const NSchemeCache::TSchemeCacheNavigate::TResultSet& resolveSet, const TActorContext &ctx) { for (const auto& entry: resolveSet) { - if (entry.Attributes.contains(DocApiTableVersionAttribute)) { + if (entry.Attributes.contains(NDocApi::VersionAttribute)) { auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Document API table cannot be modified" << ": "<< CanonizePath(entry.Path)); diff --git a/ydb/core/viewer/healthcheck_record.h b/ydb/core/viewer/healthcheck_record.h new file mode 100644 index 0000000000..39c94e2887 --- /dev/null +++ b/ydb/core/viewer/healthcheck_record.h @@ -0,0 +1,44 @@ +#pragma once + +namespace NKikimr::NViewer { + +using namespace NActors; +using namespace NMonitoring; + +struct TMetricRecord { + TString Database; + TString Message; + TString Status; + TString Type; + + bool operator!=(const TMetricRecord& x) const noexcept { + return !(x == *this); + } + + bool operator==(const TMetricRecord& x) const noexcept { + return this->Database == x.Database && this->Message == x.Message && this->Status == x.Status && this->Type == x.Type; + } + + ui64 Hash() const noexcept { + ui64 hash = std::hash<TString>()(Database); + hash = CombineHashes<ui64>(hash, std::hash<TString>()(Message)); + hash = CombineHashes<ui64>(hash, std::hash<TString>()(Status)); + hash = CombineHashes<ui64>(hash, std::hash<TString>()(Type)); + return hash; + } + + struct THash { + ui64 operator()(const TMetricRecord& record) const noexcept { + return record.Hash(); + } + }; +}; + +} + +template<> +struct THash<NKikimr::NViewer::TMetricRecord> { + inline ui64 operator()(const NKikimr::NViewer::TMetricRecord& x) const noexcept { + return x.Hash(); + } +}; diff --git a/ydb/core/viewer/json_acl.h b/ydb/core/viewer/json_acl.h index c234b04df8..fd79ea2e23 100644 --- a/ydb/core/viewer/json_acl.h +++ b/ydb/core/viewer/json_acl.h @@ -169,7 +169,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_browse.h b/ydb/core/viewer/json_browse.h index 20e1c6e119..64d423e853 100644 --- a/ydb/core/viewer/json_browse.h +++ b/ydb/core/viewer/json_browse.h @@ -189,7 +189,7 @@ public: } } TStringStream result; - result << Viewer->GetHTTPGATEWAYTIMEOUT(); + result << Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()); RenderPendingRequests(result); ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(result.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); diff --git a/ydb/core/viewer/json_bscontrollerinfo.h b/ydb/core/viewer/json_bscontrollerinfo.h index 1daafd2368..f8fd43ca65 100644 --- a/ydb/core/viewer/json_bscontrollerinfo.h +++ b/ydb/core/viewer/json_bscontrollerinfo.h @@ -67,7 +67,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_cluster.h b/ydb/core/viewer/json_cluster.h index 0bef71d6b5..e85ed63ba1 100644 --- a/ydb/core/viewer/json_cluster.h +++ b/ydb/core/viewer/json_cluster.h @@ -80,9 +80,52 @@ public: ++Requested; ctx.Send(whiteboardServiceId, new TEvWhiteboard::TEvBSGroupStateRequest(), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); ++Requested; - if (Tablets) { - ctx.Send(whiteboardServiceId, new TEvWhiteboard::TEvTabletStateRequest(), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); - ++Requested; + } + + void SendTabletStateRequest(ui32 nodeId, const TActorContext& ctx, THashSet<TTabletId>& filterTablets) { + auto request = new TEvWhiteboard::TEvTabletStateRequest(); + for (TTabletId id: filterTablets) { + request->Record.AddFilterTabletId(id); + } + TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); + ctx.Send(whiteboardServiceId, request, IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId); + ++Requested; + } + + void SendTabletStateRequest(const TActorContext& ctx) { + TIntrusivePtr<TDomainsInfo> domains = AppData(ctx)->DomainsInfo; + TIntrusivePtr<TDomainsInfo::TDomain> domain = domains->Domains.begin()->second; + THashSet<TTabletId> filterTablets; + for (TTabletId id : domain->Coordinators) { + filterTablets.emplace(id); + } + for (TTabletId id : domain->Mediators) { + filterTablets.emplace(id); + } + for (TTabletId id : domain->TxAllocators) { + filterTablets.emplace(id); + } + const NKikimrSchemeOp::TPathDescription& pathDescription(DescribeResult->GetRecord().GetPathDescription()); + if (pathDescription.HasDomainDescription()) { + const NKikimrSubDomains::TDomainDescription& domainDescription(pathDescription.GetDomainDescription()); + for (TTabletId tabletId : domainDescription.GetProcessingParams().GetCoordinators()) { + filterTablets.emplace(tabletId); + } + for (TTabletId tabletId : domainDescription.GetProcessingParams().GetMediators()) { + filterTablets.emplace(tabletId); + } + if (domainDescription.HasDomainKey()) { + if (domainDescription.GetDomainKey().HasSchemeShard()) { + filterTablets.emplace(domainDescription.GetDomainKey().GetSchemeShard()); + } + } + } + + TIntrusivePtr<TDynamicNameserviceConfig> dynamicNameserviceConfig = AppData()->DynamicNameserviceConfig; + for (const auto& ni : NodesInfo->Nodes) { + if (ni.NodeId <= dynamicNameserviceConfig->MaxStaticNodeId) { + SendTabletStateRequest(ni.NodeId, ctx, filterTablets); + } } } @@ -206,6 +249,10 @@ public: void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev, const TActorContext &ctx) { if (ev->Get()->GetRecord().GetStatus() == NKikimrScheme::StatusSuccess) { DescribeResult = ev->Release(); + + if (Tablets) { + SendTabletStateRequest(ctx); + } } RequestDone(ctx); } diff --git a/ydb/core/viewer/json_compute.h b/ydb/core/viewer/json_compute.h index b549567460..50bb45df70 100644 --- a/ydb/core/viewer/json_compute.h +++ b/ydb/core/viewer/json_compute.h @@ -47,6 +47,9 @@ class TJsonCompute : public TViewerPipeClient<TJsonCompute> { TTabletId RootHiveId = 0; bool RootHiveRequested = false; NKikimrViewer::TComputeInfo Result; + ui32 UptimeSeconds = 0; + bool ProblemNodesOnly = false; + TString Filter; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -86,6 +89,9 @@ public: Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); Tablets = FromStringWithDefault<bool>(params.Get("tablets"), Tablets); Path = params.Get("path"); + UptimeSeconds = FromStringWithDefault<ui32>(params.Get("uptime"), 0); + ProblemNodesOnly = FromStringWithDefault<bool>(params.Get("problems_only"), ProblemNodesOnly); + Filter = params.Get("filter"); SendRequest(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); @@ -261,6 +267,33 @@ public: void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr&) { } + bool CheckNodeFilters(TNodeId nodeId) { + auto itSysInfo = NodeSysInfo.find(nodeId); + if (itSysInfo != NodeSysInfo.end()) { + if (itSysInfo->second.SystemStateInfoSize() == 1) { + const NKikimrWhiteboard::TSystemStateInfo& sysInfo = itSysInfo->second.GetSystemStateInfo(0); + if (UptimeSeconds > 0 && sysInfo.HasStartTime() && sysInfo.HasChangeTime() + && sysInfo.GetChangeTime() - sysInfo.GetStartTime() > UptimeSeconds * 1000) { + return false; + } + if (ProblemNodesOnly && sysInfo.HasSystemState() + && GetViewerFlag(sysInfo.GetSystemState()) == NKikimrViewer::EFlag::Green) { + return false; + } + if (Filter) { + if (sysInfo.HasHost() && sysInfo.GetHost().Contains(Filter)) { + return true; + } + if (std::to_string(nodeId).contains(Filter)) { + return true; + } + return false; + } + } + } + return true; + } + void ReplyAndPassAway() { THashMap<TNodeId, TVector<const NKikimrWhiteboard::TTabletStateInfo*>> tabletInfoIndex; NKikimrWhiteboard::TEvTabletStateResponse tabletInfo; @@ -291,6 +324,8 @@ public: TPathId subDomainKey(itSubDomainKey->second); const NKikimrViewer::TTenant& tenantBySubDomainKey(TenantBySubDomainKey[subDomainKey]); for (TNodeId nodeId : tenantBySubDomainKey.GetNodeIds()) { + if (!CheckNodeFilters(nodeId)) + continue; NKikimrViewer::TComputeNodeInfo& computeNodeInfo = *computeTenantInfo.AddNodes(); computeNodeInfo.SetNodeId(nodeId); auto itSysInfo = NodeSysInfo.find(nodeId); @@ -406,7 +441,10 @@ struct TJsonRequestParameters<TJsonCompute> { return R"___([{"name":"path","in":"query","description":"schema path","required":false,"type":"string"}, {"name":"enums","in":"query","description":"convert enums to strings","required":false,"type":"boolean"}, {"name":"ui64","in":"query","description":"return ui64 as number","required":false,"type":"boolean"}, - {"name":"timeout","in":"query","description":"timeout in ms","required":false,"type":"integer"}])___"; + {"name":"timeout","in":"query","description":"timeout in ms","required":false,"type":"integer"}, + {"name":"uptime","in":"query","description":"return only nodes with less uptime in sec.","required":false,"type":"integer"}, + {"name":"problems_only","in":"query","description":"return only problem nodes","required":false,"type":"boolean"}, + {"name":"filter","in":"query","description":"filter nodes by id or host","required":false,"type":"string"}])___"; } }; diff --git a/ydb/core/viewer/json_content.h b/ydb/core/viewer/json_content.h index 79b230f128..09ef2300d3 100644 --- a/ydb/core/viewer/json_content.h +++ b/ydb/core/viewer/json_content.h @@ -122,7 +122,7 @@ private: } void HandleBrowseTimeout(const TActorContext& ctx) { - return SendErrorReplyAndDie(Viewer->GetHTTPGATEWAYTIMEOUT(), ctx); + return SendErrorReplyAndDie(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), ctx); } void SendErrorReplyAndDie(const TString& error, const TActorContext& ctx) { diff --git a/ydb/core/viewer/json_counters.h b/ydb/core/viewer/json_counters.h index 8e5aa527d4..a4f64a1ceb 100644 --- a/ydb/core/viewer/json_counters.h +++ b/ydb/core/viewer/json_counters.h @@ -430,7 +430,7 @@ public: } void Timeout(const TActorContext& ctx) { - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } }; diff --git a/ydb/core/viewer/json_describe.h b/ydb/core/viewer/json_describe.h index 3f6ec6b59c..9f33e3b66b 100644 --- a/ydb/core/viewer/json_describe.h +++ b/ydb/core/viewer/json_describe.h @@ -1,4 +1,5 @@ #pragma once +#include <ydb/core/tx/scheme_board/cache.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/mon.h> #include <ydb/core/base/tablet_pipe.h> @@ -13,15 +14,19 @@ namespace NViewer { using namespace NActors; using NSchemeShard::TEvSchemeShard; +using TNavigate = NSchemeCache::TSchemeCacheNavigate; class TJsonDescribe : public TViewerPipeClient<TJsonDescribe> { using TBase = TViewerPipeClient<TJsonDescribe>; IViewer* Viewer; NMon::TEvHttpInfo::TPtr Event; - TAutoPtr<TEvSchemeShard::TEvDescribeSchemeResult> DescribeResult; + TAutoPtr<TEvSchemeShard::TEvDescribeSchemeResult> SchemeShardResult; + TAutoPtr<TEvTxProxySchemeCache::TEvNavigateKeySetResult> CacheResult; + TAutoPtr<NKikimrViewer::TEvDescribeSchemeInfo> DescribeResult; TJsonSettings JsonSettings; ui32 Timeout = 0; bool ExpandSubElements = true; + int Requests = 0; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -71,30 +76,167 @@ public: request->Record.SetUserToken(Event->Get()->UserToken); SendRequest(MakeTxProxyID(), request.Release()); } + ++Requests; + + if (params.Has("path")) { + TAutoPtr<NSchemeCache::TSchemeCacheNavigate> request(new NSchemeCache::TSchemeCacheNavigate()); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; + entry.SyncVersion = false; + entry.Path = SplitPath(params.Get("path")); + request->ResultSet.emplace_back(entry); + SendRequest(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request)); + ++Requests; + } + Become(&TThis::StateRequestedDescribe, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); } STATEFN(StateRequestedDescribe) { switch (ev->GetTypeRewrite()) { hFunc(TEvSchemeShard::TEvDescribeSchemeResult, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); hFunc(TEvTabletPipe::TEvClientConnected, TBase::Handle); cFunc(TEvents::TSystem::Wakeup, HandleTimeout); } } void Handle(TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { - DescribeResult = ev->Release(); - RequestDone(); + SchemeShardResult = ev->Release(); + if (SchemeShardResult->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusSuccess) { + ReplyAndPassAway(); + } else { + RequestDone("TEvDescribeSchemeResult"); + } + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev) { + CacheResult = ev->Release(); + RequestDone("TEvNavigateKeySetResult"); + } + + void RequestDone(const char* name) { + --Requests; + if (Requests == 0) { + ReplyAndPassAway(); + } + if (Requests < 0) { + BLOG_CRIT("Requests < 0 in RequestDone(" << name << ")"); + } + } + + void FillDescription(NKikimrSchemeOp::TDirEntry* descr, ui64 schemeShardId) { + descr->SetSchemeshardId(schemeShardId); + descr->SetPathId(InvalidLocalPathId); + descr->SetParentPathId(InvalidLocalPathId); + descr->SetCreateFinished(true); + descr->SetCreateTxId(0); + descr->SetCreateStep(0); + } + + NKikimrSchemeOp::EPathType ConvertType(TNavigate::EKind navigate) { + switch (navigate) { + case TNavigate::KindSubdomain: + return NKikimrSchemeOp::EPathTypeSubDomain; + case TNavigate::KindPath: + return NKikimrSchemeOp::EPathTypeDir; + case TNavigate::KindExtSubdomain: + return NKikimrSchemeOp::EPathTypeExtSubDomain; + case TNavigate::KindTable: + return NKikimrSchemeOp::EPathTypeTable; + case TNavigate::KindOlapStore: + return NKikimrSchemeOp::EPathTypeColumnStore; + case TNavigate::KindColumnTable: + return NKikimrSchemeOp::EPathTypeColumnTable; + case TNavigate::KindRtmr: + return NKikimrSchemeOp::EPathTypeRtmrVolume; + case TNavigate::KindKesus: + return NKikimrSchemeOp::EPathTypeKesus; + case TNavigate::KindSolomon: + return NKikimrSchemeOp::EPathTypeSolomonVolume; + case TNavigate::KindTopic: + return NKikimrSchemeOp::EPathTypePersQueueGroup; + case TNavigate::KindCdcStream: + return NKikimrSchemeOp::EPathTypeCdcStream; + case TNavigate::KindSequence: + return NKikimrSchemeOp::EPathTypeSequence; + case TNavigate::KindReplication: + return NKikimrSchemeOp::EPathTypeReplication; + case TNavigate::KindBlobDepot: + return NKikimrSchemeOp::EPathTypeBlobDepot; + default: + return NKikimrSchemeOp::EPathTypeDir; + } + } + + TAutoPtr<NKikimrViewer::TEvDescribeSchemeInfo> GetSchemeShardDescribeSchemeInfo() { + TAutoPtr<NKikimrViewer::TEvDescribeSchemeInfo> result(new NKikimrViewer::TEvDescribeSchemeInfo()); + auto& record = SchemeShardResult->GetRecord(); + const auto *descriptor = NKikimrScheme::EStatus_descriptor(); + result->SetStatus(descriptor->FindValueByNumber(record.GetStatus())->name()); + result->SetReason(record.GetReason()); + result->SetPath(record.GetPath()); + result->MutablePathDescription()->CopyFrom(record.GetPathDescription()); + result->SetPathOwner(record.GetPathOwner()); + result->SetPathId(record.GetPathId()); + result->SetLastExistedPrefixPath(record.GetLastExistedPrefixPath()); + result->SetLastExistedPrefixPathId(record.GetLastExistedPrefixPathId()); + result->MutableLastExistedPrefixDescription()->CopyFrom(record.GetLastExistedPrefixDescription()); + result->SetPathOwnerId(record.GetPathOwnerId()); + result->SetSource(NKikimrViewer::TEvDescribeSchemeInfo::SchemeShard); + + return result; + } + + TAutoPtr<NKikimrViewer::TEvDescribeSchemeInfo> GetCacheDescribeSchemeInfo() { + const auto& entry = CacheResult->Request.Get()->ResultSet.front(); + const auto& path = Event->Get()->Request.GetParams().Get("path"); + const auto& pathId = TPathId(); + const auto& schemeShardId = entry.DomainInfo->DomainKey.OwnerId; + + TAutoPtr<NKikimrViewer::TEvDescribeSchemeInfo> result(new NKikimrViewer::TEvDescribeSchemeInfo()); + result->SetPath(path); + result->SetPathOwner(schemeShardId); + result->SetPathId(pathId.LocalPathId); + result->SetPathOwnerId(pathId.OwnerId); + + auto* pathDescription = result->MutablePathDescription(); + auto* self = pathDescription->MutableSelf(); + + self->CopyFrom(entry.Self->Info); + FillDescription(self, schemeShardId); + + if (entry.ListNodeEntry) { + for (const auto& child : entry.ListNodeEntry->Children) { + auto descr = pathDescription->AddChildren(); + descr->SetName(child.Name); + descr->SetPathType(ConvertType(child.Kind)); + FillDescription(descr, schemeShardId); + } + }; + const auto *descriptor = NKikimrScheme::EStatus_descriptor(); + auto status = descriptor->FindValueByNumber(NKikimrScheme::StatusSuccess)->name(); + result->SetStatus(status); + result->SetSource(NKikimrViewer::TEvDescribeSchemeInfo::Cache); + return result; } void ReplyAndPassAway() { TStringStream json; TString headers = Viewer->GetHTTPOKJSON(Event->Get()); + if (SchemeShardResult != nullptr && SchemeShardResult->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusSuccess) { + DescribeResult = GetSchemeShardDescribeSchemeInfo(); + } else if (CacheResult != nullptr) { + NSchemeCache::TSchemeCacheNavigate *navigate = CacheResult->Request.Get(); + Y_VERIFY(navigate->ResultSet.size() == 1); + if (navigate->ErrorCount == 0) { + DescribeResult = GetCacheDescribeSchemeInfo(); + } + } if (DescribeResult != nullptr) { if (ExpandSubElements) { - NKikimrScheme::TEvDescribeSchemeResult& describe = *(DescribeResult->MutableRecord()); - if (describe.HasPathDescription()) { - auto& pathDescription = *describe.MutablePathDescription(); + if (DescribeResult->HasPathDescription()) { + auto& pathDescription = *DescribeResult->MutablePathDescription(); if (pathDescription.HasTable()) { auto& table = *pathDescription.MutableTable(); for (auto& tableIndex : table.GetTableIndexes()) { @@ -110,14 +252,12 @@ public: } } } - TProtoToJson::ProtoToJson(json, DescribeResult->GetRecord(), JsonSettings); - switch (DescribeResult->GetRecord().GetStatus()) { - case NKikimrScheme::StatusAccessDenied: + const auto *descriptor = NKikimrScheme::EStatus_descriptor(); + auto accessDeniedStatus = descriptor->FindValueByNumber(NKikimrScheme::StatusAccessDenied)->name(); + if (DescribeResult->GetStatus() == accessDeniedStatus) { headers = HTTPFORBIDDENJSON; - break; - default: - break; } + TProtoToJson::ProtoToJson(json, *DescribeResult, JsonSettings); } else { json << "null"; } @@ -127,7 +267,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_handlers_viewer.cpp b/ydb/core/viewer/json_handlers_viewer.cpp index b3c7954d71..daa64a165e 100644 --- a/ydb/core/viewer/json_handlers_viewer.cpp +++ b/ydb/core/viewer/json_handlers_viewer.cpp @@ -30,12 +30,10 @@ #include "json_query.h" #include "json_netinfo.h" #include "json_compute.h" -#include "counters_hosts.h" #include "json_healthcheck.h" #include "json_nodes.h" #include "json_acl.h" - namespace NKikimr::NViewer { template <> diff --git a/ydb/core/viewer/json_healthcheck.h b/ydb/core/viewer/json_healthcheck.h index 2f0e5efa73..03abb5da30 100644 --- a/ydb/core/viewer/json_healthcheck.h +++ b/ydb/core/viewer/json_healthcheck.h @@ -9,37 +9,67 @@ #include <ydb/core/viewer/json/json.h> #include <ydb/core/health_check/health_check.h> #include <ydb/core/util/proto_duration.h> +#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <util/string/split.h> +#include "healthcheck_record.h" +#include <vector> namespace NKikimr { namespace NViewer { using namespace NActors; +enum HealthCheckResponseFormat { + JSON, + PROMETHEUS +}; + class TJsonHealthCheck : public TActorBootstrapped<TJsonHealthCheck> { static const bool WithRetry = false; - using TBase = TActorBootstrapped<TJsonHealthCheck>; - IViewer* Viewer; NMon::TEvHttpInfo::TPtr Event; TJsonSettings JsonSettings; ui32 Timeout = 0; + HealthCheckResponseFormat Format; + TString Database; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::VIEWER_HANDLER; } - TJsonHealthCheck(IViewer* viewer, NMon::TEvHttpInfo::TPtr& ev) - : Viewer(viewer) - , Event(ev) + TJsonHealthCheck(IViewer*, NMon::TEvHttpInfo::TPtr& ev) + : Event(ev) {} - void Bootstrap() { + void Bootstrap(const TActorContext& ctx) { const auto& params(Event->Get()->Request.GetParams()); - JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); - JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); + + Format = HealthCheckResponseFormat::JSON; + if (params.Has("format")) { + auto& format = params.Get("format"); + if (format == "json") { + Format = HealthCheckResponseFormat::JSON; + } else if (format == "prometheus") { + Format = HealthCheckResponseFormat::PROMETHEUS; + } + } else if (const auto *header = Event->Get()->Request.GetHeaders().FindHeader("Accept")) { + THashSet<TString> accept; + StringSplitter(header->Value()).SplitBySet(", ").SkipEmpty().Collect(&accept); + if (accept.contains("*/*") || accept.contains("application/json")) { + Format = HealthCheckResponseFormat::JSON; + } else if (accept.contains("text/plain")) { + Format = HealthCheckResponseFormat::PROMETHEUS; + } else { + Format = HealthCheckResponseFormat::JSON; + } + } + if (Format == HealthCheckResponseFormat::JSON) { + JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); + JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); + } Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); THolder<NHealthCheck::TEvSelfCheckRequest> request = MakeHolder<NHealthCheck::TEvSelfCheckRequest>(); - request->Database = params.Get("tenant"); + request->Database = Database = params.Get("tenant"); request->Request.set_return_verbose_status(FromStringWithDefault<bool>(params.Get("verbose"), false)); request->Request.set_maximum_level(FromStringWithDefault<ui32>(params.Get("max_level"), 0)); SetDuration(TDuration::MilliSeconds(Timeout), *request->Request.mutable_operation_params()->mutable_operation_timeout()); @@ -54,26 +84,110 @@ public: } Send(NHealthCheck::MakeHealthCheckID(), request.Release()); Timeout += Timeout * 20 / 100; // we prefer to wait for more (+20%) verbose timeout status from HC - Become(&TThis::StateRequestedInfo, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); + ctx.Schedule(TDuration::Seconds(10), new TEvents::TEvWakeup()); + Become(&TThis::StateRequestedInfo); } - STATEFN(StateRequestedInfo) { + STFUNC(StateRequestedInfo) { switch (ev->GetTypeRewrite()) { - hFunc(NHealthCheck::TEvSelfCheckResult, Handle); - cFunc(TEvents::TSystem::Wakeup, HandleTimeout); + HFunc(NHealthCheck::TEvSelfCheckResult, Handle); + CFunc(TEvents::TSystem::Wakeup, HandleTimeout); + } + } + + int GetIssueCount(const Ydb::Monitoring::IssueLog& issueLog) { + return issueLog.count() == 0 ? 1 : issueLog.count(); + } + + THolder<THashMap<TMetricRecord, ui32>> GetRecordCounters(NHealthCheck::TEvSelfCheckResult::TPtr& ev) { + const auto *descriptor = Ydb::Monitoring::StatusFlag_Status_descriptor(); + THashMap<TMetricRecord, ui32> recordCounters; + for (auto& log : ev->Get()->Result.issue_log()) { + TMetricRecord record { + .Database = log.location().database().name(), + .Message = log.message(), + .Status = descriptor->FindValueByNumber(log.status())->name(), + .Type = log.type() + }; + + auto it = recordCounters.find(record); + if (it != recordCounters.end()) { + it->second += GetIssueCount(log); + } else { + recordCounters[record] = GetIssueCount(log); + } } + + return MakeHolder<THashMap<TMetricRecord, ui32>>(recordCounters); } - void Handle(NHealthCheck::TEvSelfCheckResult::TPtr& ev) { + void HandleJSON(NHealthCheck::TEvSelfCheckResult::TPtr& ev, const TActorContext &ctx) { TStringStream json; TProtoToJson::ProtoToJson(json, ev->Get()->Result, JsonSettings); - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPOKJSON(Event->Get()) + json.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); - PassAway(); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(HTTPOKJSON + json.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Die(ctx); + } + + void HandlePrometheus(NHealthCheck::TEvSelfCheckResult::TPtr& ev, const TActorContext &ctx) { + auto recordCounters = GetRecordCounters(ev); + + TStringStream ss; + IMetricEncoderPtr encoder = EncoderPrometheus(&ss); + IMetricEncoder* e = encoder.Get(); + + TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo; + TIntrusivePtr<TDomainsInfo::TDomain> domain = domains->Domains.begin()->second; + auto filterDatabase = Database ? Database : "/" + domain->Name; + e->OnStreamBegin(); + if (recordCounters->size() > 0) { + for (auto& recordCounter : *recordCounters) { + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "HC_" + domain->Name); + e->OnLabel("DATABASE", recordCounter.first.Database ? recordCounter.first.Database : filterDatabase); + e->OnLabel("MESSAGE", recordCounter.first.Message); + e->OnLabel("STATUS", recordCounter.first.Status); + e->OnLabel("TYPE", recordCounter.first.Type); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), recordCounter.second); + e->OnMetricEnd(); + } + } else { + const auto *descriptor = Ydb::Monitoring::SelfCheck_Result_descriptor(); + auto result = descriptor->FindValueByNumber(ev->Get()->Result.self_check_result())->name(); + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "HC_" + domain->Name); + e->OnLabel("DATABASE", filterDatabase); + e->OnLabel("MESSAGE", result); + e->OnLabel("STATUS", result); + e->OnLabel("TYPE", "ALL"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1); + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(HTTPOKTEXT + ss.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Die(ctx); + } + + void Handle(NHealthCheck::TEvSelfCheckResult::TPtr& ev, const TActorContext &ctx) { + if (Format == HealthCheckResponseFormat::JSON) { + HandleJSON(ev, ctx); + } else { + HandlePrometheus(ev, ctx); + } } - void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); - PassAway(); + void HandleTimeout(const TActorContext &ctx) { + Send(Event->Sender, new NMon::TEvHttpInfoRes(HTTPGATEWAYTIMEOUT, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Die(ctx); } }; @@ -95,7 +209,8 @@ struct TJsonRequestParameters<TJsonHealthCheck> { {"name":"tenant","in":"query","description":"path to database","required":false,"type":"string"}, {"name":"verbose","in":"query","description":"return verbose status","required":false,"type":"boolean"}, {"name":"max_level","in":"query","description":"max depth of issues to return","required":false,"type":"integer"}, - {"name":"min_status","in":"query","description":"min status of issues to return","required":false,"type":"string"}])___"; + {"name":"min_status","in":"query","description":"min status of issues to return","required":false,"type":"string"}, + {"name":"format","in":"query","description":"format of reply","required":false,"type":"string"}])___"; } }; diff --git a/ydb/core/viewer/json_hiveinfo.h b/ydb/core/viewer/json_hiveinfo.h index 0fa58d281d..0747d047fa 100644 --- a/ydb/core/viewer/json_hiveinfo.h +++ b/ydb/core/viewer/json_hiveinfo.h @@ -96,7 +96,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_hivestats.h b/ydb/core/viewer/json_hivestats.h index 4c2e623006..b5dcd126e8 100644 --- a/ydb/core/viewer/json_hivestats.h +++ b/ydb/core/viewer/json_hivestats.h @@ -74,7 +74,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_hotkeys.h b/ydb/core/viewer/json_hotkeys.h index 714a9aa17c..b9c0839c6f 100644 --- a/ydb/core/viewer/json_hotkeys.h +++ b/ydb/core/viewer/json_hotkeys.h @@ -155,7 +155,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_labeledcounters.h b/ydb/core/viewer/json_labeledcounters.h index 3b3663bc47..065ea88da5 100644 --- a/ydb/core/viewer/json_labeledcounters.h +++ b/ydb/core/viewer/json_labeledcounters.h @@ -142,7 +142,7 @@ public: } void HandleTimeout(const TActorContext &ctx) { - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } }; diff --git a/ydb/core/viewer/json_local_rpc.h b/ydb/core/viewer/json_local_rpc.h index bab530b264..5ccee817a3 100644 --- a/ydb/core/viewer/json_local_rpc.h +++ b/ydb/core/viewer/json_local_rpc.h @@ -211,7 +211,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_metainfo.h b/ydb/core/viewer/json_metainfo.h index f507909e24..61b0f8097f 100644 --- a/ydb/core/viewer/json_metainfo.h +++ b/ydb/core/viewer/json_metainfo.h @@ -115,7 +115,7 @@ public: void HandleTimeout(const TActorContext &ctx) { TStringStream result; - result << Viewer->GetHTTPGATEWAYTIMEOUT(); + result << Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()); RenderPendingRequests(result); ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(result.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); diff --git a/ydb/core/viewer/json_netinfo.h b/ydb/core/viewer/json_netinfo.h index 8d4ab1ddee..6acc16e415 100644 --- a/ydb/core/viewer/json_netinfo.h +++ b/ydb/core/viewer/json_netinfo.h @@ -295,7 +295,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_nodes.h b/ydb/core/viewer/json_nodes.h index 7dc859d9b7..b9ce001eba 100644 --- a/ydb/core/viewer/json_nodes.h +++ b/ydb/core/viewer/json_nodes.h @@ -550,7 +550,7 @@ public: if (With == EWith::SpaceProblems) { auto itSystemState = SysInfo.find(nodeId); if (itSystemState != SysInfo.end() && itSystemState->second.SystemStateInfoSize() > 0) { - if (itSystemState->second.GetSystemStateInfo(0).GetMaxDiskUsage() < 85) { + if (itSystemState->second.GetSystemStateInfo(0).GetMaxDiskUsage() < 0.85) { continue; } } diff --git a/ydb/core/viewer/json_pqconsumerinfo.h b/ydb/core/viewer/json_pqconsumerinfo.h index b7526edf7d..6f6d420e42 100644 --- a/ydb/core/viewer/json_pqconsumerinfo.h +++ b/ydb/core/viewer/json_pqconsumerinfo.h @@ -112,7 +112,7 @@ public: } void HandleTimeout(const TActorContext &ctx) { - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } }; diff --git a/ydb/core/viewer/json_query.h b/ydb/core/viewer/json_query.h index e69e567748..869c230d7a 100644 --- a/ydb/core/viewer/json_query.h +++ b/ydb/core/viewer/json_query.h @@ -98,7 +98,7 @@ public: } } if (query.empty()) { - ReplyAndPassAway(HTTPBADREQUEST); + ReplyAndPassAway(Viewer->GetHTTPBADREQUEST(Event->Get(), {}, "Bad Request")); return; } NKikimrKqp::TQueryRequest& request = *event->Record.MutableRequest(); @@ -111,7 +111,11 @@ public: request.SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); request.SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN); request.SetKeepSession(false); - } else if (Action == "explain" || Action == "explain-ast") { + } else if (Action == "execute-data") { + request.SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + request.SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); + request.SetKeepSession(false); + } else if (Action == "explain" || Action == "explain-ast" || Action == "explain-data") { request.SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); request.SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); } else if (Action == "explain-scan") { @@ -189,7 +193,7 @@ private: case NYdb::EPrimitiveType::DyNumber: return valueParser.GetDyNumber(); case NYdb::EPrimitiveType::Uuid: - return "<uuid not implemented>"; + return valueParser.GetUuid().ToString(); } } @@ -268,7 +272,7 @@ private: } void HandleTimeout() { - ReplyAndPassAway(Viewer->GetHTTPGATEWAYTIMEOUT()); + ReplyAndPassAway(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get())); } void ReplyAndPassAway(TString data) { @@ -278,7 +282,7 @@ private: private: void MakeErrorReply(TStringBuilder& out, NJson::TJsonValue& jsonResponse, NKikimrKqp::TEvQueryResponse& record) { - out << "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n"; + out << Viewer->GetHTTPBADREQUEST(Event->Get(), "application/json"); NJson::TJsonValue& jsonIssues = jsonResponse["issues"]; // find first deepest error diff --git a/ydb/core/viewer/json_tabletcounters.h b/ydb/core/viewer/json_tabletcounters.h index 19202d95d1..86ccdd42cc 100644 --- a/ydb/core/viewer/json_tabletcounters.h +++ b/ydb/core/viewer/json_tabletcounters.h @@ -162,7 +162,7 @@ public: } void HandleTimeout(const TActorContext &ctx) { - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } }; diff --git a/ydb/core/viewer/json_tabletinfo.h b/ydb/core/viewer/json_tabletinfo.h index 66605f815d..a5c86a9469 100644 --- a/ydb/core/viewer/json_tabletinfo.h +++ b/ydb/core/viewer/json_tabletinfo.h @@ -3,6 +3,7 @@ #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/core/mon.h> #include <ydb/core/protos/services.pb.h> +#include <ydb/core/protos/node_whiteboard.pb.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> @@ -77,9 +78,11 @@ struct TWhiteboardMergerComparator<NNodeWhiteboard::TEvWhiteboard::TEvTabletStat class TJsonTabletInfo : public TJsonWhiteboardRequest<TEvWhiteboard::TEvTabletStateRequest, TEvWhiteboard::TEvTabletStateResponse> { static const bool WithRetry = false; + bool ReplyWithDeadTabletsInfo; using TBase = TJsonWhiteboardRequest<TEvWhiteboard::TEvTabletStateRequest, TEvWhiteboard::TEvTabletStateResponse>; using TThis = TJsonTabletInfo; - TVector<ui64> Tablets; + THashMap<ui64, NKikimrTabletBase::TTabletTypes::EType> Tablets; + TTabletId HiveId; public: TJsonTabletInfo(IViewer *viewer, NMon::TEvHttpInfo::TPtr &ev) : TJsonWhiteboardRequest(viewer, ev) @@ -91,7 +94,9 @@ public: void Bootstrap() override { BLOG_TRACE("Bootstrap()"); const auto& params(Event->Get()->Request.GetParams()); + ReplyWithDeadTabletsInfo = params.Has("path"); if (params.Has("path")) { + TBase::RequestSettings.Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate()); if (!Event->Get()->UserToken.empty()) { request->Record.SetUserToken(Event->Get()->UserToken); @@ -109,7 +114,7 @@ public: TString strTabletId(TBase::RequestSettings.FilterFields.substr(10, TBase::RequestSettings.FilterFields.size() - 11)); TTabletId uiTabletId(FromStringWithDefault<TTabletId>(strTabletId, {})); if (uiTabletId) { - Tablets.push_back(uiTabletId); + Tablets[uiTabletId] = NKikimrTabletBase::TTabletTypes::Unknown; Request->Record.AddFilterTabletId(uiTabletId); } } @@ -118,19 +123,21 @@ public: } void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr &ev) { + const NKikimrScheme::TEvDescribeSchemeResult &rec = ev->Get()->GetRecord(); + HiveId = rec.GetPathDescription().GetDomainDescription().GetProcessingParams().GetHive(); + THolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult> describeResult = ev->Release(); if (describeResult->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusSuccess) { const auto& pathDescription = describeResult->GetRecord().GetPathDescription(); if (pathDescription.GetSelf().GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeTable) { - Tablets.reserve(describeResult->GetRecord().GetPathDescription().TablePartitionsSize()); for (const auto& partition : describeResult->GetRecord().GetPathDescription().GetTablePartitions()) { - Tablets.emplace_back(partition.GetDatashardId()); + Tablets[partition.GetDatashardId()] = NKikimrTabletBase::TTabletTypes::DataShard; } } if (pathDescription.GetSelf().GetPathType() == NKikimrSchemeOp::EPathType::EPathTypePersQueueGroup) { Tablets.reserve(describeResult->GetRecord().GetPathDescription().GetPersQueueGroup().PartitionsSize()); for (const auto& partition : describeResult->GetRecord().GetPathDescription().GetPersQueueGroup().GetPartitions()) { - Tablets.emplace_back(partition.GetTabletId()); + Tablets[partition.GetTabletId()] = NKikimrTabletBase::TTabletTypes::PersQueue; } } if (pathDescription.GetSelf().GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeDir @@ -138,49 +145,57 @@ public: || pathDescription.GetSelf().GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeExtSubDomain) { if (pathDescription.HasDomainDescription()) { for (TTabletId tabletId : pathDescription.GetDomainDescription().GetProcessingParams().GetCoordinators()) { - Tablets.emplace_back(tabletId); + Tablets[tabletId] = NKikimrTabletBase::TTabletTypes::Coordinator; } for (TTabletId tabletId : pathDescription.GetDomainDescription().GetProcessingParams().GetMediators()) { - Tablets.emplace_back(tabletId); + Tablets[tabletId] = NKikimrTabletBase::TTabletTypes::Mediator; } if (pathDescription.GetDomainDescription().GetProcessingParams().HasSchemeShard()) { - Tablets.emplace_back(pathDescription.GetDomainDescription().GetProcessingParams().GetSchemeShard()); + Tablets[pathDescription.GetDomainDescription().GetProcessingParams().GetSchemeShard()] = NKikimrTabletBase::TTabletTypes::SchemeShard; } else { TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo; TIntrusivePtr<TDomainsInfo::TDomain> domain = domains->Domains.begin()->second; - - Tablets.emplace_back(domain->SchemeRoot); + Tablets[domain->SchemeRoot] = NKikimrTabletBase::TTabletTypes::SchemeShard; ui32 hiveDomain = domains->GetHiveDomainUid(domain->DefaultHiveUid); ui64 defaultStateStorageGroup = domains->GetDefaultStateStorageGroup(hiveDomain); - Tablets.emplace_back(MakeBSControllerID(defaultStateStorageGroup)); - Tablets.emplace_back(MakeConsoleID(defaultStateStorageGroup)); - Tablets.emplace_back(MakeNodeBrokerID(defaultStateStorageGroup)); + Tablets[MakeBSControllerID(defaultStateStorageGroup)] = NKikimrTabletBase::TTabletTypes::BSController; + Tablets[MakeConsoleID(defaultStateStorageGroup)] = NKikimrTabletBase::TTabletTypes::Console; + Tablets[MakeNodeBrokerID(defaultStateStorageGroup)] = NKikimrTabletBase::TTabletTypes::NodeBroker; } if (pathDescription.GetDomainDescription().GetProcessingParams().HasHive()) { - Tablets.emplace_back(pathDescription.GetDomainDescription().GetProcessingParams().GetHive()); + Tablets[pathDescription.GetDomainDescription().GetProcessingParams().GetHive()] = NKikimrTabletBase::TTabletTypes::Hive; } } } - Sort(Tablets); - Tablets.erase(std::unique(Tablets.begin(), Tablets.end()), Tablets.end()); } if (Tablets.empty()) { ReplyAndPassAway(); } else { - for (TTabletId tabletId : Tablets) { - Request->Record.AddFilterTabletId(tabletId); + TBase::Bootstrap(); + for (auto tablet : Tablets) { + Request->Record.AddFilterTabletId(tablet.first); } } - TBase::Bootstrap(); } virtual void FilterResponse(NKikimrWhiteboard::TEvTabletStateResponse& response) override { if (!Tablets.empty()) { NKikimrWhiteboard::TEvTabletStateResponse result; for (const NKikimrWhiteboard::TTabletStateInfo& info : response.GetTabletStateInfo()) { - if (BinarySearch(Tablets.begin(), Tablets.end(), info.GetTabletId())) { + auto tablet = Tablets.find(info.GetTabletId()); + if (tablet != Tablets.end()) { result.MutableTabletStateInfo()->Add()->CopyFrom(info); + Tablets.erase(tablet->first); + } + } + if (ReplyWithDeadTabletsInfo) { + for (auto tablet : Tablets) { + auto deadTablet = result.MutableTabletStateInfo()->Add(); + deadTablet->SetTabletId(tablet.first); + deadTablet->SetState(NKikimrWhiteboard::TTabletStateInfo::Dead); + deadTablet->SetType(tablet.second); + deadTablet->SetHiveId(HiveId); } } result.SetResponseTime(response.GetResponseTime()); diff --git a/ydb/core/viewer/json_tenantinfo.h b/ydb/core/viewer/json_tenantinfo.h index d2bf9ac35b..a7b0d8a6f9 100644 --- a/ydb/core/viewer/json_tenantinfo.h +++ b/ydb/core/viewer/json_tenantinfo.h @@ -595,6 +595,10 @@ public: tenant.SetName(path); tenant.SetOverall(overall); } + std::sort(Result.MutableTenantInfo()->begin(), Result.MutableTenantInfo()->end(), + [](const NKikimrViewer::TTenant& a, const NKikimrViewer::TTenant& b) { + return a.name() < b.name(); + }); TStringStream json; TProtoToJson::ProtoToJson(json, Result, JsonSettings); Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPOKJSON(Event->Get()) + json.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); diff --git a/ydb/core/viewer/json_tenants.h b/ydb/core/viewer/json_tenants.h index 098d15813c..94c020096c 100644 --- a/ydb/core/viewer/json_tenants.h +++ b/ydb/core/viewer/json_tenants.h @@ -95,7 +95,7 @@ public: } void HandleTimeout() { - Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); PassAway(); } }; diff --git a/ydb/core/viewer/json_topicinfo.h b/ydb/core/viewer/json_topicinfo.h index 28d79a446f..c0614b756f 100644 --- a/ydb/core/viewer/json_topicinfo.h +++ b/ydb/core/viewer/json_topicinfo.h @@ -87,7 +87,7 @@ public: } void HandleTimeout(const TActorContext &ctx) { - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } }; diff --git a/ydb/core/viewer/json_whoami.h b/ydb/core/viewer/json_whoami.h index fea374b41d..9b4468a1b1 100644 --- a/ydb/core/viewer/json_whoami.h +++ b/ydb/core/viewer/json_whoami.h @@ -44,7 +44,7 @@ public: } void HandleTimeout(const TActorContext &ctx) { - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); Die(ctx); } }; diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index 07e2ecf75e..9b8ca6324b 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -504,3 +504,24 @@ message TEvViewerResponse { NKikimrWhiteboard.TEvTabletStateResponse TabletResponse = 11; } } + +message TEvDescribeSchemeInfo { + enum ESource { + None = 0; + SchemeShard = 1; + Cache = 2; + } + + optional string Status = 1; + optional string Reason = 2; + optional string Path = 3; + optional NKikimrSchemeOp.TPathDescription PathDescription = 4; + optional fixed64 PathOwner = 5; + optional fixed64 PathId = 6; + + optional string LastExistedPrefixPath = 7; + optional fixed64 LastExistedPrefixPathId = 8; + optional NKikimrSchemeOp.TPathDescription LastExistedPrefixDescription = 9; + optional fixed64 PathOwnerId = 10; + optional ESource Source = 11; +} diff --git a/ydb/core/viewer/viewer.cpp b/ydb/core/viewer/viewer.cpp index 36dbf96f94..166ad5fd86 100644 --- a/ydb/core/viewer/viewer.cpp +++ b/ydb/core/viewer/viewer.cpp @@ -20,6 +20,7 @@ #include "browse_pq.h" #include "browse_db.h" #include "counters_hosts.h" +#include "json_healthcheck.h" #include "json_handlers.h" @@ -114,6 +115,12 @@ public: .UseAuth = false, }); mon->RegisterActorPage({ + .RelPath = "healthcheck", + .ActorSystem = ctx.ExecutorThread.ActorSystem, + .ActorId = ctx.SelfID, + .UseAuth = false, + }); + mon->RegisterActorPage({ .Title = "VDisk", .RelPath = "vdisk", .ActorSystem = ctx.ExecutorThread.ActorSystem, @@ -139,8 +146,10 @@ public: return KikimrRunConfig; } + TString GetCORS(const NMon::TEvHttpInfo* request) override; TString GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString response) override; - TString GetHTTPGATEWAYTIMEOUT() override; + TString GetHTTPGATEWAYTIMEOUT(const NMon::TEvHttpInfo* request) override; + TString GetHTTPBADREQUEST(const NMon::TEvHttpInfo* request, TString type, TString response) override; void RegisterVirtualHandler( NKikimrViewer::EObjectType parentObjectType, @@ -345,6 +354,10 @@ private: ctx.ExecutorThread.RegisterActor(new TCountersHostsList(this, ev)); return; } + if (filename.StartsWith("healthcheck")) { + ctx.ExecutorThread.RegisterActor(new TJsonHealthCheck(this, ev)); + return; + } // TODO: check path validity // TODO: cache if (msg->Request.GetPathInfo().StartsWith('/')) { @@ -416,12 +429,9 @@ IActor* CreateViewer(const TKikimrRunConfig& kikimrRunConfig) { return new TViewer(kikimrRunConfig); } -TString TViewer::GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString response) { +TString TViewer::GetCORS(const NMon::TEvHttpInfo* request) { TStringBuilder res; TString origin; - res << "HTTP/1.1 200 Ok\r\n" - << "Content-Type: application/json; charset=utf-8\r\n" - << "X-Worker-Name: " << CurrentWorkerName << "\r\n"; if (AllowOrigin) { origin = AllowOrigin; } else if (request && request->Request.GetHeaders().HasHeader("Origin")) { @@ -433,6 +443,15 @@ TString TViewer::GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString respons << "Access-Control-Allow-Headers: Content-Type,Authorization,Origin,Accept\r\n" << "Access-Control-Allow-Methods: OPTIONS, GET, POST\r\n"; } + return res; +} + +TString TViewer::GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString response) { + TStringBuilder res; + res << "HTTP/1.1 200 Ok\r\n" + << "Content-Type: application/json; charset=utf-8\r\n" + << "X-Worker-Name: " << CurrentWorkerName << "\r\n"; + res << GetCORS(request); if (response) { res << "Content-Length: " << response.size() << "\r\n"; } @@ -443,16 +462,37 @@ TString TViewer::GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString respons return res; } -TString TViewer::GetHTTPGATEWAYTIMEOUT() { - return TStringBuilder() - << "HTTP/1.1 504 Gateway Time-out\r\nConnection: Close\r\n" - << "X-Worker-Name: " << FQDNHostName() << ":" << CurrentWorkerName << "\r\n" - << "\r\nGateway Time-out\r\n"; +TString TViewer::GetHTTPGATEWAYTIMEOUT(const NMon::TEvHttpInfo* request) { + TStringBuilder res; + res << "HTTP/1.1 504 Gateway Time-out\r\n" + << "Connection: Close\r\n" + << "X-Worker-Name: " << FQDNHostName() << ":" << CurrentWorkerName << "\r\n"; + res << GetCORS(request); + res << "\r\nGateway Time-out\r\n"; + return res; +} + +TString TViewer::GetHTTPBADREQUEST(const NMon::TEvHttpInfo* request, TString contentType, TString response) { + TStringBuilder res; + res << "HTTP/1.1 400 Bad Request\r\n" + << "Connection: Close\r\n"; + if (contentType) { + res << "Content-Type: " << contentType << "\r\n"; + } + res << GetCORS(request); + res << "\r\n"; + if (response) { + res << response; + } + return res; } NKikimrViewer::EFlag GetFlagFromTabletState(NKikimrWhiteboard::TTabletStateInfo::ETabletState state) { NKikimrViewer::EFlag flag = NKikimrViewer::EFlag::Grey; switch (state) { + case NKikimrWhiteboard::TTabletStateInfo::Dead: + flag = NKikimrViewer::EFlag::Red; + break; case NKikimrWhiteboard::TTabletStateInfo::Created: case NKikimrWhiteboard::TTabletStateInfo::ResolveStateStorage: case NKikimrWhiteboard::TTabletStateInfo::Candidate: @@ -461,12 +501,7 @@ NKikimrViewer::EFlag GetFlagFromTabletState(NKikimrWhiteboard::TTabletStateInfo: case NKikimrWhiteboard::TTabletStateInfo::Restored: case NKikimrWhiteboard::TTabletStateInfo::Discover: case NKikimrWhiteboard::TTabletStateInfo::Lock: - case NKikimrWhiteboard::TTabletStateInfo::Dead: - flag = NKikimrViewer::EFlag::Red; - break; case NKikimrWhiteboard::TTabletStateInfo::RebuildGraph: - flag = NKikimrViewer::EFlag::Orange; - break; case NKikimrWhiteboard::TTabletStateInfo::ResolveLeader: flag = NKikimrViewer::EFlag::Yellow; break; diff --git a/ydb/core/viewer/viewer.h b/ydb/core/viewer/viewer.h index 3d0971696b..fd51cb53ae 100644 --- a/ydb/core/viewer/viewer.h +++ b/ydb/core/viewer/viewer.h @@ -153,8 +153,10 @@ public: NKikimrViewer::EObjectType objectType, const TContentHandler& handler) = 0; - virtual TString GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString response = {} ) = 0; - virtual TString GetHTTPGATEWAYTIMEOUT() = 0; + virtual TString GetCORS(const NMon::TEvHttpInfo* request) = 0; + virtual TString GetHTTPOKJSON(const NMon::TEvHttpInfo* request, TString response = {}) = 0; + virtual TString GetHTTPGATEWAYTIMEOUT(const NMon::TEvHttpInfo* request) = 0; + virtual TString GetHTTPBADREQUEST(const NMon::TEvHttpInfo* request, TString contentType = {}, TString response = {}) = 0; }; void SetupPQVirtualHandlers(IViewer* viewer); @@ -186,7 +188,6 @@ static const char HTTPOKTEXT[] = "HTTP/1.1 200 Ok\r\nAccess-Control-Allow-Origin static const char HTTPFORBIDDENJSON[] = "HTTP/1.1 403 Forbidden\r\nAccess-Control-Allow-Origin: *\r\nContent-Type: application/json; charset=utf-8\r\nConnection: Close\r\n\r\n"; static const char HTTPGATEWAYTIMEOUT[] = "HTTP/1.1 504 Gateway Time-out\r\nConnection: Close\r\n\r\nGateway Time-out\r\n"; static const char HTTPBADREQUEST[] = "HTTP/1.1 400 Bad Request\r\nConnection: Close\r\n\r\nBad Request\r\n"; -static const char HTTPBADREQUEST_HEADERS[] = "HTTP/1.1 400 Bad Request\r\nConnection: Close\r\n\r\n"; template <typename ValueType, typename OutputIteratorType> void GenericSplitIds(TStringBuf source, char delim, OutputIteratorType it) { diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 68cd1ec619..ebed1fa558 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -121,8 +121,14 @@ void FillColumnDescriptionImpl(TYdbProto& out, } } - if (in.HasTTLSettings() && in.GetTTLSettings().HasEnabled()) { - AddTtl(out, in.GetTTLSettings().GetEnabled()); + if (in.HasTTLSettings()) { + if (in.GetTTLSettings().HasEnabled()) { + AddTtl(out, in.GetTTLSettings().GetEnabled()); + } + + if (in.GetTTLSettings().HasUseTiering()) { + out.set_tiering(in.GetTTLSettings().GetUseTiering()); + } } } @@ -159,8 +165,14 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc } } - if (in.HasTtlSettings() && in.GetTtlSettings().HasEnabled()) { - AddTtl(out, in.GetTtlSettings().GetEnabled()); + if (in.HasTtlSettings()) { + if (in.GetTtlSettings().HasEnabled()) { + AddTtl(out, in.GetTtlSettings().GetEnabled()); + } + + if (in.GetTtlSettings().HasUseTiering()) { + out.set_tiering(in.GetTtlSettings().GetUseTiering()); + } } } @@ -413,6 +425,18 @@ bool FillIndexDescription(NKikimrSchemeOp::TIndexedTableCreationConfig& out, return true; } +template <typename TOutProto, typename TInProto> +void FillAttributesImpl(TOutProto& out, const TInProto& in) { + if (!in.UserAttributesSize()) { + return; + } + + auto& outAttrs = *out.mutable_attributes(); + for (const auto& inAttr : in.GetUserAttributes()) { + outAttrs[inAttr.GetKey()] = inAttr.GetValue(); + } +} + void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { @@ -421,6 +445,7 @@ void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, changefeed->set_name(stream.GetName()); changefeed->set_virtual_timestamps(stream.GetVirtualTimestamps()); + changefeed->set_aws_region(stream.GetAwsRegion()); switch (stream.GetMode()) { case NKikimrSchemeOp::ECdcStreamMode::ECdcStreamModeKeysOnly: @@ -438,6 +463,9 @@ void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, case NKikimrSchemeOp::ECdcStreamFormat::ECdcStreamFormatJson: changefeed->set_format(Ydb::Table::ChangefeedFormat::FORMAT_JSON); break; + case NKikimrSchemeOp::ECdcStreamFormat::ECdcStreamFormatDynamoDBStreamsJson: + changefeed->set_format(Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON); + break; default: break; } @@ -451,6 +479,8 @@ void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, default: break; } + + FillAttributesImpl(*changefeed, stream); } } @@ -459,6 +489,7 @@ bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, out.SetName(in.name()); out.SetVirtualTimestamps(in.virtual_timestamps()); + out.SetAwsRegion(in.aws_region()); switch (in.mode()) { case Ydb::Table::ChangefeedMode::MODE_KEYS_ONLY: @@ -478,6 +509,9 @@ bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, case Ydb::Table::ChangefeedFormat::FORMAT_JSON: out.SetFormat(NKikimrSchemeOp::ECdcStreamFormat::ECdcStreamFormatJson); break; + case Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON: + out.SetFormat(NKikimrSchemeOp::ECdcStreamFormat::ECdcStreamFormatDynamoDBStreamsJson); + break; default: status = Ydb::StatusIds::BAD_REQUEST; error = "Invalid changefeed format"; @@ -493,6 +527,12 @@ bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, out.SetState(NKikimrSchemeOp::ECdcStreamState::ECdcStreamStateScan); } + for (const auto& [key, value] : in.attributes()) { + auto& attr = *out.AddUserAttributes(); + attr.SetKey(key); + attr.SetValue(value); + } + return true; } @@ -677,20 +717,6 @@ void FillColumnFamilies(Ydb::Table::CreateTableRequest& out, FillColumnFamiliesImpl(out, in); } -template <typename TYdbProto> -void FillAttributesImpl(TYdbProto& out, - const NKikimrSchemeOp::TPathDescription& in) { - - if (!in.UserAttributesSize()) { - return; - } - - auto& outAttrs = *out.mutable_attributes(); - for (const auto& inAttr : in.GetUserAttributes()) { - outAttrs[inAttr.GetKey()] = inAttr.GetValue(); - } -} - void FillAttributes(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TPathDescription& in) { FillAttributesImpl(out, in); diff --git a/ydb/core/ydb_convert/table_settings.cpp b/ydb/core/ydb_convert/table_settings.cpp index 0e5b56d85f..76040ec87a 100644 --- a/ydb/core/ydb_convert/table_settings.cpp +++ b/ydb/core/ydb_convert/table_settings.cpp @@ -196,6 +196,10 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, } } + if (proto.tiering().size()) { + tableDesc.MutableTTLSettings()->SetUseTiering(proto.tiering()); + } + return true; } diff --git a/ydb/core/ymq/queues/fifo/queries.cpp b/ydb/core/ymq/queues/fifo/queries.cpp index ecc24b93eb..24e4a07ae4 100644 --- a/ydb/core/ymq/queues/fifo/queries.cpp +++ b/ydb/core/ymq/queues/fifo/queries.cpp @@ -549,7 +549,7 @@ const char* const ListQueuesQuery = R"__( (let queuesRange '( '('Account userName userName) '('QueueName (Utf8String '"") (Void)))) - (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards)) + (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards 'TablesFormat)) (let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List)) (let filtered (Filter queues (lambda '(item) (block '( diff --git a/ydb/core/ymq/queues/std/queries.cpp b/ydb/core/ymq/queues/std/queries.cpp index 9d7339fa3b..4036ec881a 100644 --- a/ydb/core/ymq/queues/std/queries.cpp +++ b/ydb/core/ymq/queues/std/queries.cpp @@ -573,7 +573,7 @@ const char* const ListQueuesQuery = R"__( (let queuesRange '( '('Account userName userName) '('QueueName (Utf8String '"") (Void)))) - (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards)) + (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards 'TablesFormat)) (let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List)) (let filtered (Filter queues (lambda '(item) (block '( diff --git a/ydb/library/testlib/service_mocks/access_service_mock.h b/ydb/library/testlib/service_mocks/access_service_mock.h index 486f36b63b..8de34b8de2 100644 --- a/ydb/library/testlib/service_mocks/access_service_mock.h +++ b/ydb/library/testlib/service_mocks/access_service_mock.h @@ -76,29 +76,39 @@ public: THashSet<TString> AllowedUserTokens = {"user1"}; THashMap<TString, TString> AllowedServiceTokens = {{"service1", "root1/folder1"}}; + bool ShouldGenerateRetryableError = false; + grpc::Status Authenticate( grpc::ServerContext*, const yandex::cloud::priv::servicecontrol::v1::AuthenticateRequest* request, yandex::cloud::priv::servicecontrol::v1::AuthenticateResponse* response) override { ++AuthenticateCount; - TString token = request->iam_token(); - if (InvalidTokens.count(token) > 0) { - return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid Token"); - } - if (UnavailableTokens.count(token) > 0) { - return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service Unavailable"); - } - if (AllowedUserTokens.count(token) > 0) { - response->mutable_subject()->mutable_user_account()->set_id(token); - return grpc::Status::OK; - } - if (AllowedServiceTokens.count(token) > 0) { - response->mutable_subject()->mutable_service_account()->set_id(token); - response->mutable_subject()->mutable_service_account()->set_folder_id(AllowedServiceTokens[token]); + if (request->has_signature()) { + if (ShouldGenerateRetryableError) { + return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service Unavailable"); + } + response->mutable_subject()->mutable_user_account()->set_id("user1"); return grpc::Status::OK; + } else { + TString token = request->iam_token(); + if (InvalidTokens.count(token) > 0) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid Token"); + } + if (UnavailableTokens.count(token) > 0) { + return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service Unavailable"); + } + if (AllowedUserTokens.count(token) > 0) { + response->mutable_subject()->mutable_user_account()->set_id(token); + return grpc::Status::OK; + } + if (AllowedServiceTokens.count(token) > 0) { + response->mutable_subject()->mutable_service_account()->set_id(token); + response->mutable_subject()->mutable_service_account()->set_folder_id(AllowedServiceTokens[token]); + return grpc::Status::OK; + } + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Access Denied"); } - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Access Denied"); } THashSet<TString> AllowedUserPermissions = {"user1-something.read"}; @@ -111,30 +121,38 @@ public: const yandex::cloud::priv::servicecontrol::v1::AuthorizeRequest* request, yandex::cloud::priv::servicecontrol::v1::AuthorizeResponse* response) override { ++AuthorizeCount; - TString token = request->iam_token(); - if (UnavailableUserPermissions.count(token + '-' + request->permission()) > 0) { - return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service Unavailable"); - } - bool allowedResource = true; - if (!AllowedResourceIds.empty()) { - allowedResource = false; - for (const auto& resourcePath : request->resource_path()) { - if (AllowedResourceIds.count(resourcePath.id()) > 0) { - allowedResource = true; - } + if (request->has_signature()) { + if (ShouldGenerateRetryableError) { + return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service Unavailable"); } - } - if (allowedResource) { - if (AllowedUserPermissions.count(token + '-' + request->permission()) > 0) { - response->mutable_subject()->mutable_user_account()->set_id(token); - return grpc::Status::OK; + response->mutable_subject()->mutable_user_account()->set_id("user1"); + return grpc::Status::OK; + } else { + TString token = request->iam_token(); + if (UnavailableUserPermissions.count(token + '-' + request->permission()) > 0) { + return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service Unavailable"); } - if (AllowedServicePermissions.count(token + '-' + request->permission()) > 0) { - response->mutable_subject()->mutable_service_account()->set_id(token); - response->mutable_subject()->mutable_service_account()->set_folder_id(AllowedServicePermissions[token + '-' + request->permission()]); - return grpc::Status::OK; + bool allowedResource = true; + if (!AllowedResourceIds.empty()) { + allowedResource = false; + for (const auto& resourcePath : request->resource_path()) { + if (AllowedResourceIds.count(resourcePath.id()) > 0) { + allowedResource = true; + } + } + } + if (allowedResource) { + if (AllowedUserPermissions.count(token + '-' + request->permission()) > 0) { + response->mutable_subject()->mutable_user_account()->set_id(token); + return grpc::Status::OK; + } + if (AllowedServicePermissions.count(token + '-' + request->permission()) > 0) { + response->mutable_subject()->mutable_service_account()->set_id(token); + response->mutable_subject()->mutable_service_account()->set_folder_id(AllowedServicePermissions[token + '-' + request->permission()]); + return grpc::Status::OK; + } } + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Access Denied"); } - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Access Denied"); } }; diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 1b972e6e0f..4c69fb789b 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -4,6 +4,7 @@ #include <ydb/core/base/domain.h> #include <ydb/core/erasure/erasure.h> +#include <library/cpp/json/writer/json.h> namespace NKikimr::NYaml { @@ -219,6 +220,37 @@ namespace NKikimr::NYaml { return it->second; } + const NJson::TJsonArray::TArray& GetTabletIdsFor(NJson::TJsonValue& json, TString type) { + auto& systemTabletsConfig = json["system_tablets"]; + TString toLowerType = to_lower(type); + + if (!systemTabletsConfig.Has(toLowerType)) { + auto& stubs = systemTabletsConfig[toLowerType]; + stubs.SetType(NJson::EJsonValueType::JSON_ARRAY); + for(ui32 idx = 0; idx < GetDefaultTabletCount(type); ++idx) { + NJson::TJsonValue stub; + stub.SetType(NJson::EJsonValueType::JSON_MAP); + stub.InsertValue("type", type); + + stubs.AppendValue(std::move(stub)); + } + } + + ui32 idx = 0; + for(NJson::TJsonValue& tablet : systemTabletsConfig[toLowerType].GetArraySafe()) { + ++idx; + + NJson::TJsonValue& tabletInfo = tablet["info"]; + + if (!tabletInfo.Has("tablet_id")) { + Y_ENSURE_BT(idx <= GetDefaultTabletCount(type)); + tabletInfo.InsertValue("tablet_id", NJson::TJsonValue(GetNextTabletID(type, idx))); + } + } + + return json["system_tablets"][toLowerType].GetArraySafe(); + } + const NJson::TJsonArray::TArray& GetTabletsFor(NJson::TJsonValue& json, TString type) { auto& systemTabletsConfig = json["system_tablets"]; TString toLowerType = to_lower(type); @@ -349,6 +381,7 @@ namespace NKikimr::NYaml { Y_ENSURE_BT(serviceSet.Has("groups"), "groups field should be specified in service_set field of blob_storage_config"); auto& groups = serviceSet["groups"]; + bool shouldFillVdisks = !serviceSet.Has("vdisks"); auto& vdisksServiceSet = serviceSet["vdisks"]; if (shouldFillVdisks) { @@ -492,7 +525,11 @@ namespace NKikimr::NYaml { } } - void PrepareSystemTabletsInfo(NJson::TJsonValue& json) { + void PrepareSystemTabletsInfo(NJson::TJsonValue& json, bool relaxed) { + if (relaxed && (!json.Has("nameservice_config") || !json["nameservice_config"].Has("node"))) { + return; + } + if (!json.Has("system_tablets")) { auto& config = json["system_tablets"]; config.SetType(NJson::EJsonValueType::JSON_MAP); @@ -512,11 +549,15 @@ namespace NKikimr::NYaml { } - void PrepareBootstrapConfig(NJson::TJsonValue& json) { + void PrepareBootstrapConfig(NJson::TJsonValue& json, bool relaxed) { if (json.Has("bootstrap_config") && json["bootstrap_config"].Has("tablet")) { return; } + if (relaxed && (!json.Has("system_tablets") || !json.Has("static_erasure"))) { + return; + } + if (!json.Has("bootstrap_config")) { auto& bootstrapConfig = json["bootstrap_config"]; bootstrapConfig.SetType(NJson::EJsonValueType::JSON_MAP); @@ -532,7 +573,10 @@ namespace NKikimr::NYaml { } } - void PrepareDomainsConfig(NJson::TJsonValue& json) { + void PrepareDomainsConfig(NJson::TJsonValue& json, bool relaxed) { + if (relaxed && !json.Has("domains_config")) { + return; + } Y_ENSURE_BT(json.Has("domains_config")); Y_ENSURE_BT(json["domains_config"].IsMap()); @@ -586,17 +630,24 @@ namespace NKikimr::NYaml { const std::vector<std::pair<TString, TString>> exps = {{"explicit_coordinators", "FLAT_TX_COORDINATOR"}, {"explicit_allocators", "TX_ALLOCATOR"}, {"explicit_mediators", "TX_MEDIATOR"}}; for(auto [field, type] : exps) { + if (relaxed && domain.Has(field)) { + continue; + } Y_ENSURE_BT(!domain.Has(field)); auto& arr = domain[field]; arr.SetType(NJson::EJsonValueType::JSON_ARRAY); - for(auto tablet: GetTabletsFor(json, type)) { + for(auto tablet: GetTabletIdsFor(json, type)) { arr.AppendValue(GetUnsignedIntegerSafe(tablet["info"], "tablet_id")); } } } } - void PrepareSecurityConfig(NJson::TJsonValue& json) { + void PrepareSecurityConfig(NJson::TJsonValue& json, bool relaxed) { + if (relaxed && !json.Has("domains_config")) { + return; + } + Y_ENSURE_BT(json.Has("domains_config")); Y_ENSURE_BT(json["domains_config"].IsMap()); @@ -783,16 +834,16 @@ namespace NKikimr::NYaml { } } - void PrepareJson(NJson::TJsonValue& json){ + void TransformConfig(NJson::TJsonValue& json, bool relaxed) { PrepareNameserviceConfig(json); PrepareActorSystemConfig(json); PrepareStaticGroup(json); PrepareIcConfig(json); PrepareLogConfig(json); - PrepareSystemTabletsInfo(json); - PrepareDomainsConfig(json); - PrepareSecurityConfig(json); - PrepareBootstrapConfig(json); + PrepareSystemTabletsInfo(json, relaxed); + PrepareDomainsConfig(json, relaxed); + PrepareSecurityConfig(json, relaxed); + PrepareBootstrapConfig(json, relaxed); ClearFields(json); } @@ -833,7 +884,7 @@ namespace NKikimr::NYaml { void Parse(const TString& data, NKikimrConfig::TAppConfig& config) { auto yamlNode = YAML::Load(data); NJson::TJsonValue jsonNode = Yaml2Json(yamlNode, true); - PrepareJson(jsonNode); + TransformConfig(jsonNode); NProtobufJson::MergeJson2Proto(jsonNode, config, GetJsonToProtoConfig()); } } diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h index 52dd59350d..ac6a293b1a 100644 --- a/ydb/library/yaml_config/yaml_config_parser.h +++ b/ydb/library/yaml_config/yaml_config_parser.h @@ -17,5 +17,7 @@ namespace NKikimr::NYaml { NKikimrBlobStorage::TConfigRequest BuildInitDistributedStorageCommand(const TString& data); + void TransformConfig(NJson::TJsonValue& config, bool relaxed = false); + void Parse(const TString& data, NKikimrConfig::TAppConfig& config); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp index ebd1b6fafd..256d757577 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp @@ -86,11 +86,11 @@ struct TExpandedRange { TExpandedRangeBoundary Right; }; -TExpandedRangeBoundary Max(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare::TPtr cmp) { +TExpandedRangeBoundary Max(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare* cmp) { return cmp->Less(a.Value, b.Value) ? b : a; } -TExpandedRangeBoundary Min(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare::TPtr cmp) { +TExpandedRangeBoundary Min(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare* cmp) { return cmp->Less(a.Value, b.Value) ? a : b; } @@ -256,8 +256,8 @@ bool RangeCanMerge(const TExpandedRange& a, const TExpandedRange& b, const TRang bool rightIncluded = rights.back().Get<i32>() != 0; for (size_t i = 0; i < lefts.size() - 1; i += 2) { - auto infCmp = typeInfo.ComponentsCompare[i]; - auto compCmp = typeInfo.ComponentsCompare[i + 1]; + auto infCmp = typeInfo.ComponentsCompare[i].Get(); + auto compCmp = typeInfo.ComponentsCompare[i + 1].Get(); auto infCompareRes = infCmp->Compare(lefts[i], rights[i]); Y_ENSURE(infCompareRes <= 0); @@ -383,7 +383,7 @@ public: for (size_t i = 1; i < mergedLists.size(); ++i) { auto toUnion = ExpandRange(mergedLists[i]); if (RangeCanMerge(current, toUnion, TypeInfos.front())) { - current = { current.Left, Max(current.Right, toUnion.Right, TypeInfos.front().BoundaryCompare) }; + current = { current.Left, Max(current.Right, toUnion.Right, TypeInfos.front().BoundaryCompare.Get()) }; TUnboxedValueVector newValue = { current.Left.Value, current.Right.Value }; unionList.back() = ctx.HolderFactory.VectorAsArray(newValue); } else { @@ -437,8 +437,8 @@ private: void DoIntersect(TComputationContext& ctx, TUnboxedValueQueue& current, TUnboxedValueQueue&& next) const { TUnboxedValueQueue result; - auto cmp = TypeInfos.front().RangeCompare; - auto boundaryCmp = TypeInfos.front().BoundaryCompare; + auto cmp = TypeInfos.front().RangeCompare.Get(); + auto boundaryCmp = TypeInfos.front().BoundaryCompare.Get(); while (!current.empty() && !next.empty()) { TUnboxedValueQueue* minInput; TUnboxedValueQueue* maxInput; @@ -455,7 +455,7 @@ private: TExpandedRange intersected; intersected.Left = maxRange.Left; - intersected.Right = Min(minRange.Right, maxRange.Right, TypeInfos.front().BoundaryCompare); + intersected.Right = Min(minRange.Right, maxRange.Right, TypeInfos.front().BoundaryCompare.Get()); if (!RangeIsEmpty(intersected, TypeInfos.front())) { TUnboxedValueVector newValue = { intersected.Left.Value, intersected.Right.Value }; result.push_back(ctx.HolderFactory.VectorAsArray(newValue)); @@ -492,7 +492,11 @@ public: } TUnboxedValueQueue current = std::move(expandedLists.front()); - std::vector<ICompare::TPtr> currentComponentsCompare = TypeInfos.front().ComponentsCompare; + std::vector<ICompare*> currentComponentsCompare; + currentComponentsCompare.reserve(TypeInfos.front().ComponentsCompare.size()); + for (const auto& comp : TypeInfos.front().ComponentsCompare) { + currentComponentsCompare.push_back(comp.Get()); + } for (size_t i = 1; i < expandedLists.size(); ++i) { if (expandedLists[i].empty()) { return ctx.HolderFactory.GetEmptyContainer(); @@ -516,7 +520,7 @@ private: } bool DoMultiply(TComputationContext& ctx, ui64 limit, TUnboxedValueQueue& current, const TUnboxedValueQueue& next, - std::vector<ICompare::TPtr>& currentCmps, const TRangeTypeInfo& nextTypeInfo) const + std::vector<ICompare*>& currentCmps, const TRangeTypeInfo& nextTypeInfo) const { TUnboxedValueQueue result; Y_ENSURE(currentCmps.size() >= 3 && currentCmps.size() % 2 == 1); @@ -538,15 +542,16 @@ private: } } - ICompare::TPtr currentLast = currentCmps.back(); currentCmps.pop_back(); - currentCmps.insert(currentCmps.end(), nextTypeInfo.ComponentsCompare.begin(), nextTypeInfo.ComponentsCompare.end()); + for (const auto& comp : nextTypeInfo.ComponentsCompare) { + currentCmps.push_back(comp.Get()); + } std::swap(current, result); return true; } - static bool RangeIsPoint(const TExpandedRange& range, const std::vector<ICompare::TPtr>& cmps) { + static bool RangeIsPoint(const TExpandedRange& range, const std::vector<ICompare*>& cmps) { Y_ENSURE(range.Left.Components.size() == cmps.size()); TUnboxedValue leftIncluded = range.Left.Components.back(); TUnboxedValue rightIncluded = range.Right.Components.back(); diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index d08d593db7..644be4d03f 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -458,11 +458,7 @@ private: name == "KqpWideReadTable" || name == "KqpWideReadTableRanges" || name == "KqpLookupTable" || - name == "KqpReadTable" || - name == "RangeMultiply" || - name == "RangeUnion" || - name == "RangeIntersect" || - name == "RangeFinalize" + name == "KqpReadTable" ) { PatternNodes->SuitableForCache = false; } diff --git a/ydb/library/yql/minikql/dom/make.cpp b/ydb/library/yql/minikql/dom/make.cpp index e28c089589..ead949e404 100644 --- a/ydb/library/yql/minikql/dom/make.cpp +++ b/ydb/library/yql/minikql/dom/make.cpp @@ -35,7 +35,7 @@ TUnboxedValuePod MakeData(const TDataTypeId nodeType, const TUnboxedValuePod val Y_FAIL("Unsupported data type."); } -TUnboxedValuePod MakeList(const ITypeInfoHelper::TPtr typeHelper, const TType* itemType, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { +TUnboxedValuePod MakeList(const ITypeInfoHelper* typeHelper, const TType* itemType, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { if (const auto elements = value.GetElements()) { if (const auto size = value.GetListLength()) { TUnboxedValue* items = nullptr; @@ -63,7 +63,7 @@ TUnboxedValuePod MakeList(const ITypeInfoHelper::TPtr typeHelper, const TType* i return SetNodeType<ENodeType::List>(TUnboxedValuePod::Void()); } -TUnboxedValuePod MakeDict(const ITypeInfoHelper::TPtr typeHelper, const TType* itemType, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { +TUnboxedValuePod MakeDict(const ITypeInfoHelper* typeHelper, const TType* itemType, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { TSmallVec<TPair, TStdAllocatorForUdf<TPair>> items; items.reserve(value.GetDictLength()); const auto it = value.GetDictIterator(); @@ -78,7 +78,7 @@ TUnboxedValuePod MakeDict(const ITypeInfoHelper::TPtr typeHelper, const TType* i return SetNodeType<ENodeType::Dict>(TUnboxedValuePod(new TMapNode(items.data(), items.size()))); } -TUnboxedValuePod MakeTuple(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { +TUnboxedValuePod MakeTuple(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { if (const auto tupleTypeInspector = TTupleTypeInspector(*typeHelper, shape); const auto size = tupleTypeInspector.GetElementsCount()) { TUnboxedValue* items = nullptr; auto res = valueBuilder->NewArray(size, items); @@ -91,7 +91,7 @@ TUnboxedValuePod MakeTuple(const ITypeInfoHelper::TPtr typeHelper, const TType* return SetNodeType<ENodeType::List>(TUnboxedValuePod::Void()); } -TUnboxedValuePod MakeStruct(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { +TUnboxedValuePod MakeStruct(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { if (const auto structTypeInspector = TStructTypeInspector(*typeHelper, shape); const auto size = structTypeInspector.GetMembersCount()) { TSmallVec<TPair, TStdAllocatorForUdf<TPair>> items; items.reserve(size); @@ -109,7 +109,7 @@ TUnboxedValuePod MakeStruct(const ITypeInfoHelper::TPtr typeHelper, const TType* return SetNodeType<ENodeType::Dict>(TUnboxedValuePod::Void()); } -TUnboxedValuePod MakeVariant(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { +TUnboxedValuePod MakeVariant(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { const auto index = value.GetVariantIndex(); const auto& item = value.GetVariantItem(); const auto underlyingType = TVariantTypeInspector(*typeHelper, shape).GetUnderlyingType(); @@ -130,7 +130,7 @@ TUnboxedValuePod MakeVariant(const ITypeInfoHelper::TPtr typeHelper, const TType } -TUnboxedValuePod MakeDom(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { +TUnboxedValuePod MakeDom(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder) { switch (const auto kind = typeHelper->GetTypeKind(shape)) { case ETypeKind::Null: return MakeEntity(); diff --git a/ydb/library/yql/minikql/dom/make.h b/ydb/library/yql/minikql/dom/make.h index a6bba77819..3cfaabd62f 100644 --- a/ydb/library/yql/minikql/dom/make.h +++ b/ydb/library/yql/minikql/dom/make.h @@ -5,6 +5,6 @@ namespace NYql::NDom { -NUdf::TUnboxedValuePod MakeDom(const NUdf::ITypeInfoHelper::TPtr typeHelper, const NUdf::TType* shape, const NUdf::TUnboxedValuePod value, const NUdf::IValueBuilder* valueBuilder); +NUdf::TUnboxedValuePod MakeDom(const NUdf::ITypeInfoHelper* typeHelper, const NUdf::TType* shape, const NUdf::TUnboxedValuePod value, const NUdf::IValueBuilder* valueBuilder); } diff --git a/ydb/library/yql/minikql/dom/peel.cpp b/ydb/library/yql/minikql/dom/peel.cpp index 7f996688b8..44a27aa46b 100644 --- a/ydb/library/yql/minikql/dom/peel.cpp +++ b/ydb/library/yql/minikql/dom/peel.cpp @@ -37,10 +37,10 @@ TUnboxedValuePod PeelData(const TDataTypeId nodeType, const TUnboxedValuePod val } template<bool Strict, bool AutoConvert> -TUnboxedValuePod TryPeelDom(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); +TUnboxedValuePod TryPeelDom(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); template<bool Strict, bool AutoConvert> -TUnboxedValuePod PeelList(const ITypeInfoHelper::TPtr typeHelper, const TType* itemType, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod PeelList(const ITypeInfoHelper* typeHelper, const TType* itemType, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { switch (GetNodeType(x)) { case ENodeType::List: { if (!x.IsBoxed()) @@ -87,7 +87,7 @@ TUnboxedValuePod PeelList(const ITypeInfoHelper::TPtr typeHelper, const TType* i } template<bool Strict, bool AutoConvert, bool Utf8Keys> -TUnboxedValuePod PeelDict(const ITypeInfoHelper::TPtr typeHelper, const TType* itemType, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod PeelDict(const ITypeInfoHelper* typeHelper, const TType* itemType, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { switch (GetNodeType(x)) { case ENodeType::Dict: if (!x.IsBoxed()) @@ -130,7 +130,7 @@ TUnboxedValuePod PeelDict(const ITypeInfoHelper::TPtr typeHelper, const TType* i return valueBuilder->NewEmptyList().Release(); } -TUnboxedValuePod MakeStub(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod MakeStub(const ITypeInfoHelper* typeHelper, const TType* shape, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { switch (const auto kind = typeHelper->GetTypeKind(shape)) { case ETypeKind::Optional: return TUnboxedValuePod(); @@ -191,7 +191,7 @@ TUnboxedValuePod MakeStub(const ITypeInfoHelper::TPtr typeHelper, const TType* s } template<bool Strict, bool AutoConvert> -TUnboxedValuePod PeelTuple(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod PeelTuple(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { if (const auto tupleTypeInspector = TTupleTypeInspector(*typeHelper, shape); auto count = tupleTypeInspector.GetElementsCount()) { switch (GetNodeType(x)) { case ENodeType::List: { @@ -254,7 +254,7 @@ TUnboxedValuePod PeelTuple(const ITypeInfoHelper::TPtr typeHelper, const TType* } template<bool Strict, bool AutoConvert> -TUnboxedValuePod PeelStruct(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod PeelStruct(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { if (const auto structTypeInspector = TStructTypeInspector(*typeHelper, shape)) { const auto size = structTypeInspector.GetMembersCount(); switch (GetNodeType(x)) { @@ -307,7 +307,7 @@ TUnboxedValuePod PeelStruct(const ITypeInfoHelper::TPtr typeHelper, const TType* } template<bool Strict, bool AutoConvert> -TUnboxedValuePod PeelOptional(const ITypeInfoHelper::TPtr typeHelper, const TType* itemType, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod PeelOptional(const ITypeInfoHelper* typeHelper, const TType* itemType, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { if (IsNodeType<ENodeType::Entity>(value)) return TUnboxedValuePod().MakeOptional(); @@ -320,7 +320,7 @@ TUnboxedValuePod PeelOptional(const ITypeInfoHelper::TPtr typeHelper, const TTyp } template<bool Strict, bool AutoConvert> -TUnboxedValuePod TryPeelDom(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod TryPeelDom(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { switch (const auto kind = typeHelper->GetTypeKind(shape)) { case ETypeKind::Data: return PeelData<Strict, AutoConvert>(TDataTypeInspector(*typeHelper, shape).GetTypeId(), value, valueBuilder, pos); @@ -356,7 +356,7 @@ TUnboxedValuePod TryPeelDom(const ITypeInfoHelper::TPtr typeHelper, const TType* } template<bool Strict, bool AutoConvert> -TUnboxedValuePod PeelDom(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { +TUnboxedValuePod PeelDom(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos) { if (const auto result = TryPeelDom<Strict, AutoConvert>(typeHelper, shape, value, valueBuilder, pos)) return result.GetOptionalValue(); ::TStringBuilder sb; @@ -365,9 +365,9 @@ TUnboxedValuePod PeelDom(const ITypeInfoHelper::TPtr typeHelper, const TType* sh UdfTerminate(sb.c_str()); } -template TUnboxedValuePod PeelDom<true, true>(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); -template TUnboxedValuePod PeelDom<false, true>(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); -template TUnboxedValuePod PeelDom<true, false>(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); -template TUnboxedValuePod PeelDom<false, false>(const ITypeInfoHelper::TPtr typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); +template TUnboxedValuePod PeelDom<true, true>(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); +template TUnboxedValuePod PeelDom<false, true>(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); +template TUnboxedValuePod PeelDom<true, false>(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); +template TUnboxedValuePod PeelDom<false, false>(const ITypeInfoHelper* typeHelper, const TType* shape, const TUnboxedValuePod value, const IValueBuilder* valueBuilder, const TSourcePosition& pos); } diff --git a/ydb/library/yql/minikql/dom/peel.h b/ydb/library/yql/minikql/dom/peel.h index 35e476250f..37302e59b9 100644 --- a/ydb/library/yql/minikql/dom/peel.h +++ b/ydb/library/yql/minikql/dom/peel.h @@ -6,6 +6,6 @@ namespace NYql::NDom { template<bool Strict, bool AutoConvert> -NUdf::TUnboxedValuePod PeelDom(const NUdf::ITypeInfoHelper::TPtr typeHelper, const NUdf::TType* shape, const NUdf::TUnboxedValuePod value, const NUdf::IValueBuilder* valueBuilder, const NUdf::TSourcePosition& pos); +NUdf::TUnboxedValuePod PeelDom(const NUdf::ITypeInfoHelper* typeHelper, const NUdf::TType* shape, const NUdf::TUnboxedValuePod value, const NUdf::IValueBuilder* valueBuilder, const NUdf::TSourcePosition& pos); } diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index ec613cd5cc..3b6db37450 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -1177,6 +1177,7 @@ namespace NSQLTranslationV1 { TNodePtr InitialScan; TNodePtr VirtualTimestamps; TNodePtr RetentionPeriod; + TNodePtr AwsRegion; std::optional<std::variant<TLocalSinkSettings>> SinkSettings; }; diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index beb67703fb..9cf580a27e 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -133,6 +133,9 @@ static INode::TPtr CreateChangefeedDesc(const TChangefeedDescription& desc, cons if (desc.Settings.RetentionPeriod) { settings = node.L(settings, node.Q(node.Y(node.Q("retention_period"), desc.Settings.RetentionPeriod))); } + if (desc.Settings.AwsRegion) { + settings = node.L(settings, node.Q(node.Y(node.Q("aws_region"), desc.Settings.AwsRegion))); + } if (const auto& sink = desc.Settings.SinkSettings) { switch (sink->index()) { case 0: // local diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index fafa57f747..d031f9ca26 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -1851,6 +1851,12 @@ static bool ChangefeedSettingsEntry(const TRule_changefeed_settings_entry& node, return false; } settings.RetentionPeriod = exprNode; + } else if (to_lower(id.Name) == "aws_region") { + if (!exprNode->IsLiteral() || exprNode->GetLiteralType() != "String") { + ctx.Context().Error() << "Literal of String type is expected for " << id.Name; + return false; + } + settings.AwsRegion = exprNode; } else { ctx.Context().Error(id.Pos) << "Unknown changefeed setting: " << id.Name; return false; diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index f35080c29b..1894fb54d6 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -1711,7 +1711,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { FORMAT = 'json', INITIAL_SCAN = TRUE, VIRTUAL_TIMESTAMPS = FALSE, - RETENTION_PERIOD = Interval("P1D") + RETENTION_PERIOD = Interval("P1D"), + AWS_REGION = 'aws:region' ) ); )"); @@ -1729,6 +1730,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("virtual_timestamps")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("false")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("retention_period")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws_region")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws:region")); } }; @@ -3419,6 +3422,19 @@ select FormatType($f()); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:99: Error: Literal of Interval type is expected for RETENTION_PERIOD\n"); } + Y_UNIT_TEST(InvalidChangefeedAwsRegion) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", AWS_REGION = true) + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:93: Error: Literal of String type is expected for AWS_REGION\n"); + } + Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { auto req = "USE plato;\n" "\n" diff --git a/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp b/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp index 36a90f5a70..1785a046ec 100644 --- a/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp +++ b/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp @@ -858,7 +858,7 @@ protected: class TFrom: public TBase { TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const final { - return MakeDom(TypeHelper_, Shape_, *args, valueBuilder); + return MakeDom(TypeHelper_.Get(), Shape_, *args, valueBuilder); } public: static const TStringRef& Name() { @@ -926,9 +926,9 @@ public: class TConvert: public TBase { TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const final { if (const auto options = ParseOptions(args[1]); options.Strict) - return (options.AutoConvert ? &PeelDom<true, true> : &PeelDom<true, false>)(TypeHelper_, Shape_, args[0], valueBuilder, Pos_); + return (options.AutoConvert ? &PeelDom<true, true> : &PeelDom<true, false>)(TypeHelper_.Get(), Shape_, args[0], valueBuilder, Pos_); else - return (options.AutoConvert ? &PeelDom<false, true> : &PeelDom<false, false>)(TypeHelper_, Shape_, args[0], valueBuilder, Pos_); + return (options.AutoConvert ? &PeelDom<false, true> : &PeelDom<false, false>)(TypeHelper_.Get(), Shape_, args[0], valueBuilder, Pos_); } public: diff --git a/ydb/public/api/grpc/draft/CMakeLists.darwin.txt b/ydb/public/api/grpc/draft/CMakeLists.darwin.txt index a9321e08c5..6193badda1 100644 --- a/ydb/public/api/grpc/draft/CMakeLists.darwin.txt +++ b/ydb/public/api/grpc/draft/CMakeLists.darwin.txt @@ -27,7 +27,7 @@ target_proto_messages(api-grpc-draft PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_logstore_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_query_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_topic_tx_v1.proto - ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_console_v1.proto + ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto ) target_proto_addincls(api-grpc-draft ./ diff --git a/ydb/public/api/grpc/draft/CMakeLists.linux-aarch64.txt b/ydb/public/api/grpc/draft/CMakeLists.linux-aarch64.txt index f84c8b8acf..267dd27012 100644 --- a/ydb/public/api/grpc/draft/CMakeLists.linux-aarch64.txt +++ b/ydb/public/api/grpc/draft/CMakeLists.linux-aarch64.txt @@ -28,7 +28,7 @@ target_proto_messages(api-grpc-draft PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_logstore_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_query_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_topic_tx_v1.proto - ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_console_v1.proto + ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto ) target_proto_addincls(api-grpc-draft ./ diff --git a/ydb/public/api/grpc/draft/CMakeLists.linux.txt b/ydb/public/api/grpc/draft/CMakeLists.linux.txt index f84c8b8acf..267dd27012 100644 --- a/ydb/public/api/grpc/draft/CMakeLists.linux.txt +++ b/ydb/public/api/grpc/draft/CMakeLists.linux.txt @@ -28,7 +28,7 @@ target_proto_messages(api-grpc-draft PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_logstore_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_query_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_topic_tx_v1.proto - ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_console_v1.proto + ${CMAKE_SOURCE_DIR}/ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto ) target_proto_addincls(api-grpc-draft ./ diff --git a/ydb/public/api/grpc/draft/ydb_console_v1.proto b/ydb/public/api/grpc/draft/ydb_console_v1.proto deleted file mode 100644 index 81b1be840a..0000000000 --- a/ydb/public/api/grpc/draft/ydb_console_v1.proto +++ /dev/null @@ -1,17 +0,0 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package Ydb.Console.V1; - -option java_package = "com.yandex.ydb.console.v1"; - -import "ydb/public/api/protos/draft/ydb_console.proto"; - -service ConsoleService { - rpc ApplyConfig(Console.ApplyConfigRequest) returns (Console.ApplyConfigResponse); - rpc AddVolatileConfig(Console.AddVolatileConfigRequest) returns (Console.AddVolatileConfigResponse); - rpc RemoveVolatileConfig(Console.RemoveVolatileConfigRequest) returns (Console.RemoveVolatileConfigResponse); - rpc GetConfig(Console.GetConfigRequest) returns (Console.GetConfigResponse); - rpc ResolveConfig(Console.ResolveConfigRequest) returns (Console.ResolveConfigResponse); - rpc ResolveAllConfig(Console.ResolveAllConfigRequest) returns (Console.ResolveAllConfigResponse); -} diff --git a/ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto b/ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto new file mode 100644 index 0000000000..3aa52d63e0 --- /dev/null +++ b/ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.DynamicConfig.V1; + +option java_package = "com.yandex.ydb.dynamic_config.v1"; + +import "ydb/public/api/protos/draft/ydb_dynamic_config.proto"; + +service DynamicConfigService { + // Apply new version of config. + // Overwrites previous version. + // This call is idempotent: + // - Two subsequent identical calls will return success, + // - After applying next version another identical call will result in an error. + // The field version within the YAML in the request must strictly be set to the current version increment by one. + // The field cluster within the YAML should be identical to the one configured on the node used by the console tablet. + rpc SetConfig(DynamicConfig.SetConfigRequest) returns (DynamicConfig.SetConfigResponse); + + rpc ReplaceConfig(DynamicConfig.ReplaceConfigRequest) returns (DynamicConfig.ReplaceConfigResponse); + + // Get current configs metadata. + rpc GetMetadata(DynamicConfig.GetMetadataRequest) returns (DynamicConfig.GetMetadataResponse); + + // Get current configs. + rpc GetConfig(DynamicConfig.GetConfigRequest) returns (DynamicConfig.GetConfigResponse); + + // Drop current config. + // This call is idempotent. + rpc DropConfig(DynamicConfig.DropConfigRequest) returns (DynamicConfig.DropConfigResponse); + rpc AddVolatileConfig(DynamicConfig.AddVolatileConfigRequest) returns (DynamicConfig.AddVolatileConfigResponse); + rpc RemoveVolatileConfig(DynamicConfig.RemoveVolatileConfigRequest) returns (DynamicConfig.RemoveVolatileConfigResponse); + + rpc GetNodeLabels(DynamicConfig.GetNodeLabelsRequest) returns (DynamicConfig.GetNodeLabelsResponse); + + // Resolve config for particular labels. + rpc ResolveConfig(DynamicConfig.ResolveConfigRequest) returns (DynamicConfig.ResolveConfigResponse); + rpc ResolveAllConfig(DynamicConfig.ResolveAllConfigRequest) returns (DynamicConfig.ResolveAllConfigResponse); +} diff --git a/ydb/public/api/protos/CMakeLists.darwin.txt b/ydb/public/api/protos/CMakeLists.darwin.txt index 668f29bb6c..a2284957e0 100644 --- a/ydb/public/api/protos/CMakeLists.darwin.txt +++ b/ydb/public/api/protos/CMakeLists.darwin.txt @@ -25,7 +25,7 @@ target_proto_messages(api-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_logstore.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_query.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/fq.proto - ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_console.proto + ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_dynamic_config.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/federation_discovery.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/persqueue_error_codes_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/ydb_auth.proto diff --git a/ydb/public/api/protos/CMakeLists.linux-aarch64.txt b/ydb/public/api/protos/CMakeLists.linux-aarch64.txt index 8ab433dfbe..7a83641a53 100644 --- a/ydb/public/api/protos/CMakeLists.linux-aarch64.txt +++ b/ydb/public/api/protos/CMakeLists.linux-aarch64.txt @@ -26,7 +26,7 @@ target_proto_messages(api-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_logstore.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_query.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/fq.proto - ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_console.proto + ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_dynamic_config.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/federation_discovery.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/persqueue_error_codes_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/ydb_auth.proto diff --git a/ydb/public/api/protos/CMakeLists.linux.txt b/ydb/public/api/protos/CMakeLists.linux.txt index 8ab433dfbe..7a83641a53 100644 --- a/ydb/public/api/protos/CMakeLists.linux.txt +++ b/ydb/public/api/protos/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_proto_messages(api-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_logstore.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_query.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/fq.proto - ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_console.proto + ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/draft/ydb_dynamic_config.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/federation_discovery.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/persqueue_error_codes_v1.proto ${CMAKE_SOURCE_DIR}/ydb/public/api/protos/ydb_auth.proto diff --git a/ydb/public/api/protos/draft/ydb_console.proto b/ydb/public/api/protos/draft/ydb_console.proto deleted file mode 100644 index a81fc180ee..0000000000 --- a/ydb/public/api/protos/draft/ydb_console.proto +++ /dev/null @@ -1,84 +0,0 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package Ydb.Console; -option java_package = "com.yandex.ydb.console"; - -import "ydb/public/api/protos/ydb_operation.proto"; - -message ApplyConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - optional string config = 2; -} - -message ApplyConfigResponse { - Ydb.Operations.Operation operation = 1; -} - -message AddVolatileConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - optional uint64 id = 2; - optional string config = 3; - optional uint64 version = 4; - optional string cluster = 5; -} - -message AddVolatileConfigResponse { - Ydb.Operations.Operation operation = 1; -} - -message VolatileConfig { - optional uint64 id = 1; - optional string config = 2; -} - -message GetConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; -} - -message GetConfigResponse { - Ydb.Operations.Operation operation = 1; - optional string config = 2; - repeated VolatileConfig volatile_configs = 3; - optional string cluster = 4; - optional uint32 version = 5; -} - -message RemoveVolatileConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - optional uint64 version = 2; - optional string cluster = 3; - repeated uint64 ids = 4; -} - -message RemoveVolatileConfigResponse { - Ydb.Operations.Operation operation = 1; -} - -message YamlLabel { - optional string label = 1; - optional string value = 2; -} - -message ResolveConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - optional string config = 2; - repeated VolatileConfig volatile_configs = 3; - repeated YamlLabel labels = 4; -} - -message ResolveConfigResponse { - Ydb.Operations.Operation operation = 1; - optional string config = 2; -} - -message ResolveAllConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - optional string config = 2; - repeated VolatileConfig volatile_configs = 3; -} - -message ResolveAllConfigResponse { - Ydb.Operations.Operation operation = 1; - optional string config = 2; -} diff --git a/ydb/public/api/protos/draft/ydb_dynamic_config.proto b/ydb/public/api/protos/draft/ydb_dynamic_config.proto new file mode 100644 index 0000000000..32058547bf --- /dev/null +++ b/ydb/public/api/protos/draft/ydb_dynamic_config.proto @@ -0,0 +1,221 @@ +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.DynamicConfig; +option java_package = "com.yandex.ydb.dynamic_console"; + +import "ydb/public/api/protos/ydb_operation.proto"; + +message ConfigIdentity { + // Current main config version + uint64 version = 1; + // Cluster name (should be set on node with console tablet, unknown by default) + string cluster = 2; +} + +message SetConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Config in YAML format. metadata will be ignored + string config = 2; + bool dry_run = 3; +} + +message SetConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +message ReplaceConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Config in YAML format + string config = 2; + bool dry_run = 3; +} + +message ReplaceConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +message DropConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + + ConfigIdentity identity = 2; +} + +message DropConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +message AddVolatileConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Config id must strictly be set to the current max config id increment by one. + string config = 2; +} + +message AddVolatileConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +message VolatileConfig { + // Config id + uint64 id = 1; + // Volatile config YAML document + string config = 2; +} + +message GetConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; +} + +message GetConfigResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message GetConfigResult { + // Main dynamic config with metadata in YAML format + string config = 1; + // All volatile configs + repeated VolatileConfig volatile_configs = 2; + + ConfigIdentity identity = 3; +} + +message VolatileConfigMetadata { + // Config id + uint64 id = 1; + // Volatile config YAML document + string metadata = 2; +} + +message GetMetadataRequest { + Ydb.Operations.OperationParams operation_params = 1; +} + +message GetMetadataResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message GetMetadataResult { + // Main dynamic config metadata in YAML format + string metadata = 1; + // All volatile configs + repeated VolatileConfigMetadata volatile_configs = 2; +} + +message RemoveVolatileConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + + message IdsToDelete { + repeated uint64 ids = 1; + } + + oneof consistency_check { + ConfigIdentity identity = 2; + // Ignore version and cluster checks + bool force = 3; + } + + // Ids to delete + oneof selector { + // Explicit list of ids + IdsToDelete ids = 4; + // Delete all + bool all = 5; + } +} + +message RemoveVolatileConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +message GetNodeLabelsRequest { + Ydb.Operations.OperationParams operation_params = 1; + uint32 node_id = 2; +} + +message YamlLabel { + // Name of the label + string label = 1; + // Value of the label + string value = 2; +} + +message GetNodeLabelsResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message GetNodeLabelsResult { + // Labels of a node + repeated YamlLabel labels = 1; +} + +message ResolveConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Config to resolve + string config = 2; + // Additional configs which will be apended to selectors + repeated VolatileConfig volatile_configs = 3; + repeated YamlLabel labels = 4; +} + +message ResolveConfigResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message ResolveConfigResult { + // YAML document with resolved config + string config = 1; +} + +message ResolveAllConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Config to resolve + string config = 2; + // Additional configs which will be apended to selectors + repeated VolatileConfig volatile_configs = 3; + // Either send YAML stream in config field or separate configs in configs field + bool verbose_response = 4; +} + +message YamlLabelExt { + enum LabelType { + UNSPECIFIED = 0; + COMMON = 1; + NOT_SET = 2; + EMPTY = 3; + } + // Name of the label + string label = 1; + // Type of label value + LabelType type = 2; + // Value of the label + optional string value = 3; +} + +message LabelSet { + // labels for which the config are applicable + repeated YamlLabelExt labels = 1; +} + +message ResolvedConfig { + // label sets for which the config are applicable + repeated LabelSet label_sets = 1; + // resolved YAML config + string config = 2; +} + +message ResolveAllConfigResponse { + // Result of request will be inside operation. + Ydb.Operations.Operation operation = 1; +} + +message ResolveAllConfigResult { + // YAML stream with resolved configs + // Label combinations shown in comments + string config = 1; + // Verbose resolved configs + repeated ResolvedConfig configs = 2; +} diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index 9405f11612..c8360fd7c8 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -137,7 +137,10 @@ message ChangefeedMode { message ChangefeedFormat { enum Format { FORMAT_UNSPECIFIED = 0; + // Change record in JSON format for common (row oriented) tables FORMAT_JSON = 1; + // Change record in JSON format for document (DynamoDB-compatible) tables + FORMAT_DYNAMODB_STREAMS_JSON = 2; } } @@ -154,6 +157,10 @@ message Changefeed { bool virtual_timestamps = 5; // Initial scan will output the current state of the table first bool initial_scan = 6; + // Attributes. Total size is limited to 10 KB. + map<string, string> attributes = 7 [(map_key).length.range = {min: 1, max: 100}, (length).range = {min: 1, max: 4096}]; + // Value that will be emitted in the `awsRegion` field of the record in DYNAMODB_STREAMS_JSON format + string aws_region = 8 [(length).le = 128]; } message ChangefeedDescription { @@ -182,6 +189,10 @@ message ChangefeedDescription { State state = 4; // State of emitting of virtual timestamps along with data bool virtual_timestamps = 5; + // Attributes + map<string, string> attributes = 6; + // Value that will be emitted in the `awsRegion` field of the record in DYNAMODB_STREAMS_JSON format + string aws_region = 7; } message StoragePool { @@ -498,7 +509,7 @@ message CreateTableRequest { Ydb.FeatureFlag.Status key_bloom_filter = 16; // Read replicas settings for table ReadReplicasSettings read_replicas_settings = 17; - // Tiering name for table + // Tiering rules name. It specifies how data migrates from one tier (logical storage) to another. string tiering = 18; } @@ -695,6 +706,8 @@ message DescribeTableResult { ReadReplicasSettings read_replicas_settings = 14; // List of changefeeds repeated ChangefeedDescription changefeeds = 15; + // Tiering rules name + string tiering = 16; } message Query { diff --git a/ydb/public/lib/deprecated/kicli/configurator.cpp b/ydb/public/lib/deprecated/kicli/configurator.cpp index f256c230b8..f3a28b4a3f 100644 --- a/ydb/public/lib/deprecated/kicli/configurator.cpp +++ b/ydb/public/lib/deprecated/kicli/configurator.cpp @@ -63,9 +63,10 @@ TConfigurationResult TNodeConfigurator::SyncGetNodeConfig(ui32 nodeId, const TString &nodeType, const TString& domain, const TString& token, - bool serveYaml) const + bool serveYaml, + ui64 version) const { - auto future = Kikimr->GetNodeConfig(nodeId, host, tenant, nodeType, domain, token, serveYaml); + auto future = Kikimr->GetNodeConfig(nodeId, host, tenant, nodeType, domain, token, serveYaml, version); auto result = future.GetValue(TDuration::Max()); return TConfigurationResult(result); } diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index 4dc256da18..83f7e65661 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -699,7 +699,8 @@ public: const TString &nodeType, const TString& domain = "", const TString& token = "", - bool serveYaml = false) const; + bool serveYaml = false, + ui64 version = 0) const; private: TNodeConfigurator(TKikimr& kikimr); @@ -805,7 +806,8 @@ protected: const TString &nodeType, const TString& domain, const TString& token = TString(), - bool serveYaml = false); + bool serveYaml = false, + ui64 version = 0); template <typename T> static void DumpRequest(const T& pb) { if (DUMP_REQUESTS) { diff --git a/ydb/public/lib/deprecated/kicli/kikimr.cpp b/ydb/public/lib/deprecated/kicli/kikimr.cpp index 052ae193d4..2c2761538f 100644 --- a/ydb/public/lib/deprecated/kicli/kikimr.cpp +++ b/ydb/public/lib/deprecated/kicli/kikimr.cpp @@ -599,11 +599,13 @@ NThreading::TFuture<TResult> TKikimr::GetNodeConfig(ui32 nodeId, const TString &nodeType, const TString& domain, const TString& token, - bool serveYaml) + bool serveYaml, + ui64 version) { TAutoPtr<NMsgBusProxy::TBusConsoleRequest> request = new NMsgBusProxy::TBusConsoleRequest; auto &rec = request->Record; rec.MutableGetNodeConfigRequest()->SetServeYaml(serveYaml); + rec.MutableGetNodeConfigRequest()->SetYamlApiVersion(version); auto &node = *rec.MutableGetNodeConfigRequest()->MutableNode(); node.SetNodeId(nodeId); node.SetHost(host); diff --git a/ydb/public/lib/scheme_types/scheme_type_id.h b/ydb/public/lib/scheme_types/scheme_type_id.h index 8c2aa7d2d5..c50e129ece 100644 --- a/ydb/public/lib/scheme_types/scheme_type_id.h +++ b/ydb/public/lib/scheme_types/scheme_type_id.h @@ -47,6 +47,7 @@ static constexpr TTypeId Text = Utf8; static constexpr TTypeId Yson = NYql::NProto::Yson; static constexpr TTypeId Json = NYql::NProto::Json; +static constexpr TTypeId Uuid = NYql::NProto::Uuid; static constexpr TTypeId JsonDocument = NYql::NProto::JsonDocument; static constexpr TTypeId DyNumber = NYql::NProto::DyNumber; diff --git a/ydb/public/lib/value/CMakeLists.darwin.txt b/ydb/public/lib/value/CMakeLists.darwin.txt index 50e7125445..3fac4cef8b 100644 --- a/ydb/public/lib/value/CMakeLists.darwin.txt +++ b/ydb/public/lib/value/CMakeLists.darwin.txt @@ -16,6 +16,7 @@ target_link_libraries(public-lib-value PUBLIC ydb-core-protos library-mkql_proto-protos public-lib-scheme_types + cpp-client-ydb_value ) target_sources(public-lib-value PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/lib/value/value.cpp diff --git a/ydb/public/lib/value/CMakeLists.linux-aarch64.txt b/ydb/public/lib/value/CMakeLists.linux-aarch64.txt index d77df46d22..f306cfb91c 100644 --- a/ydb/public/lib/value/CMakeLists.linux-aarch64.txt +++ b/ydb/public/lib/value/CMakeLists.linux-aarch64.txt @@ -17,6 +17,7 @@ target_link_libraries(public-lib-value PUBLIC ydb-core-protos library-mkql_proto-protos public-lib-scheme_types + cpp-client-ydb_value ) target_sources(public-lib-value PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/lib/value/value.cpp diff --git a/ydb/public/lib/value/CMakeLists.linux.txt b/ydb/public/lib/value/CMakeLists.linux.txt index d77df46d22..f306cfb91c 100644 --- a/ydb/public/lib/value/CMakeLists.linux.txt +++ b/ydb/public/lib/value/CMakeLists.linux.txt @@ -17,6 +17,7 @@ target_link_libraries(public-lib-value PUBLIC ydb-core-protos library-mkql_proto-protos public-lib-scheme_types + cpp-client-ydb_value ) target_sources(public-lib-value PRIVATE ${CMAKE_SOURCE_DIR}/ydb/public/lib/value/value.cpp diff --git a/ydb/public/lib/value/value.cpp b/ydb/public/lib/value/value.cpp index 0ed8b82689..7edde8c7a5 100644 --- a/ydb/public/lib/value/value.cpp +++ b/ydb/public/lib/value/value.cpp @@ -8,6 +8,7 @@ #include <util/string/cast.h> #include <util/string/escape.h> #include <util/string/printf.h> +#include <ydb/public/sdk/cpp/client/ydb_value/value.h> namespace NKikimr { namespace NClient { @@ -421,7 +422,13 @@ TString TValue::GetDataText() const { return ToString(Value.GetInt64()); case NScheme::NTypeIds::JsonDocument: return "\"<JsonDocument>\""; + case NScheme::NTypeIds::Uuid: + { + NYdb::TUuidValue val(Value.GetLow128(), Value.GetHi128()); + return val.ToString(); + } } + return TStringBuilder() << "\"<unknown type " << Type.GetData().GetScheme() << ">\""; } diff --git a/ydb/public/sdk/cpp/client/draft/CMakeLists.darwin.txt b/ydb/public/sdk/cpp/client/draft/CMakeLists.darwin.txt index 7a3aed1551..87ab0d4dd9 100644 --- a/ydb/public/sdk/cpp/client/draft/CMakeLists.darwin.txt +++ b/ydb/public/sdk/cpp/client/draft/CMakeLists.darwin.txt @@ -19,7 +19,7 @@ target_link_libraries(cpp-client-draft PUBLIC cpp-client-ydb_value ) target_sources(cpp-client-draft PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_console.cpp + ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_long_tx.cpp ) diff --git a/ydb/public/sdk/cpp/client/draft/CMakeLists.linux-aarch64.txt b/ydb/public/sdk/cpp/client/draft/CMakeLists.linux-aarch64.txt index fe2f75f943..591e23d3b0 100644 --- a/ydb/public/sdk/cpp/client/draft/CMakeLists.linux-aarch64.txt +++ b/ydb/public/sdk/cpp/client/draft/CMakeLists.linux-aarch64.txt @@ -20,7 +20,7 @@ target_link_libraries(cpp-client-draft PUBLIC cpp-client-ydb_value ) target_sources(cpp-client-draft PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_console.cpp + ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_long_tx.cpp ) diff --git a/ydb/public/sdk/cpp/client/draft/CMakeLists.linux.txt b/ydb/public/sdk/cpp/client/draft/CMakeLists.linux.txt index fe2f75f943..591e23d3b0 100644 --- a/ydb/public/sdk/cpp/client/draft/CMakeLists.linux.txt +++ b/ydb/public/sdk/cpp/client/draft/CMakeLists.linux.txt @@ -20,7 +20,7 @@ target_link_libraries(cpp-client-draft PUBLIC cpp-client-ydb_value ) target_sources(cpp-client-draft PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_console.cpp + ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp ${CMAKE_SOURCE_DIR}/ydb/public/sdk/cpp/client/draft/ydb_long_tx.cpp ) diff --git a/ydb/public/sdk/cpp/client/draft/ydb_console.cpp b/ydb/public/sdk/cpp/client/draft/ydb_console.cpp deleted file mode 100644 index 4985201b6f..0000000000 --- a/ydb/public/sdk/cpp/client/draft/ydb_console.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "ydb_console.h" - -#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> -#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> - -namespace NYdb::NConsole { - -class TConsoleClient::TImpl : public TClientImplCommon<TConsoleClient::TImpl> { -public: - TImpl(std::shared_ptr<TGRpcConnectionsImpl> connections) - : TClientImplCommon(std::move(connections), TConsoleClientSettings{}) - { - } - - TAsyncStatus ApplyConfig(const TString& config, const TClusterConfigSettings& settings = {}) { - auto request = MakeOperationRequest<Ydb::Console::ApplyConfigRequest>(settings); - request.set_config(config); - - return RunSimple<Ydb::Console::V1::ConsoleService, Ydb::Console::ApplyConfigRequest, Ydb::Console::ApplyConfigResponse>( - std::move(request), - &Ydb::Console::V1::ConsoleService::Stub::AsyncApplyConfig, - TRpcRequestSettings::Make(settings)); - } - - TAsyncStatus AddVolatileConfig(const TString& config, ui64 id, ui64 version, const TString& cluster, const TClusterConfigSettings& settings = {}) { - auto request = MakeOperationRequest<Ydb::Console::AddVolatileConfigRequest>(settings); - request.set_config(config); - request.set_id(id); - request.set_version(version); - request.set_cluster(cluster); - - return RunSimple<Ydb::Console::V1::ConsoleService, Ydb::Console::AddVolatileConfigRequest, Ydb::Console::AddVolatileConfigResponse>( - std::move(request), - &Ydb::Console::V1::ConsoleService::Stub::AsyncAddVolatileConfig, - TRpcRequestSettings::Make(settings)); - } - - TAsyncStatus RemoveVolatileConfig(const TString& cluster, ui64 version, const TVector<ui64>& ids, const TClusterConfigSettings& settings = {}) { - auto request = MakeOperationRequest<Ydb::Console::RemoveVolatileConfigRequest>(settings); - - request.set_cluster(cluster); - request.set_version(version); - - for (auto& id: ids) { - request.add_ids(id); - } - - return RunSimple<Ydb::Console::V1::ConsoleService, Ydb::Console::RemoveVolatileConfigRequest, Ydb::Console::RemoveVolatileConfigResponse>( - std::move(request), - &Ydb::Console::V1::ConsoleService::Stub::AsyncRemoveVolatileConfig, - TRpcRequestSettings::Make(settings)); - } - - TAsyncGetConfigResult GetConfig(const TClusterConfigSettings& settings = {}) { - auto request = MakeOperationRequest<Ydb::Console::GetConfigRequest>(settings); - - auto promise = NThreading::NewPromise<TGetConfigResult>(); - - auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - TString clusterName = "<unknown>"; - ui64 version = 0; - TString config; - TMap<ui64, TString> volatileConfigs; - if (Ydb::Console::GetConfigResponse result; any && any->UnpackTo(&result)) { - clusterName = result.cluster(); - version = result.version(); - config = result.config(); - for (const auto& config : result.volatile_configs()) { - volatileConfigs.emplace(config.id(), config.config()); - } - } - - TGetConfigResult val(TStatus(std::move(status)), std::move(clusterName), version, std::move(config), std::move(volatileConfigs)); - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred<Ydb::Console::V1::ConsoleService, Ydb::Console::GetConfigRequest, Ydb::Console::GetConfigResponse>( - std::move(request), - extractor, - &Ydb::Console::V1::ConsoleService::Stub::AsyncGetConfig, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings)); - - return promise.GetFuture(); - } - - TAsyncResolveConfigResult ResolveConfig(const TString& config, const TMap<ui64, TString>& volatileConfigs, const TMap<TString, TString>& labels, const TClusterConfigSettings& settings = {}) { - auto request = MakeOperationRequest<Ydb::Console::ResolveConfigRequest>(settings); - request.set_config(config); - for (auto& [id, volatileConfig] : volatileConfigs) { - auto* proto = request.add_volatile_configs(); - proto->set_id(id); - proto->set_config(volatileConfig); - } - for (auto& [name, value] : labels) { - auto* proto = request.add_labels(); - proto->set_label(name); - proto->set_value(value); - } - - auto promise = NThreading::NewPromise<TResolveConfigResult>(); - - auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - TString config; - if (Ydb::Console::ResolveConfigResponse result; any && any->UnpackTo(&result)) { - config = result.config(); - } - - TResolveConfigResult val(TStatus(std::move(status)), std::move(config)); - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred<Ydb::Console::V1::ConsoleService, Ydb::Console::ResolveConfigRequest, Ydb::Console::ResolveConfigResponse>( - std::move(request), - extractor, - &Ydb::Console::V1::ConsoleService::Stub::AsyncResolveConfig, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings)); - - return promise.GetFuture(); - } - - TAsyncResolveConfigResult ResolveConfig(const TString& config, const TMap<ui64, TString>& volatileConfigs, const TClusterConfigSettings& settings = {}) { - auto request = MakeOperationRequest<Ydb::Console::ResolveAllConfigRequest>(settings); - request.set_config(config); - for (auto& [id, volatileConfig] : volatileConfigs) { - auto* proto = request.add_volatile_configs(); - proto->set_id(id); - proto->set_config(volatileConfig); - } - - auto promise = NThreading::NewPromise<TResolveConfigResult>(); - - auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - TString config; - if (Ydb::Console::ResolveAllConfigResponse result; any && any->UnpackTo(&result)) { - config = result.config(); - } - - TResolveConfigResult val(TStatus(std::move(status)), std::move(config)); - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred<Ydb::Console::V1::ConsoleService, Ydb::Console::ResolveAllConfigRequest, Ydb::Console::ResolveAllConfigResponse>( - std::move(request), - extractor, - &Ydb::Console::V1::ConsoleService::Stub::AsyncResolveAllConfig, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings)); - - return promise.GetFuture(); - } -}; - -TConsoleClient::TConsoleClient(const TDriver& driver) - : Impl_(new TConsoleClient::TImpl(CreateInternalInterface(driver))) -{} - -TAsyncStatus TConsoleClient::ApplyConfig( - const TString& config, - const TClusterConfigSettings& settings) { - return Impl_->ApplyConfig(config, settings); -} - -TAsyncStatus TConsoleClient::AddVolatileConfig( - const TString& config, - ui64 id, - ui64 version, - const TString& cluster, - const TClusterConfigSettings& settings) { - return Impl_->AddVolatileConfig(config, id, version, cluster, settings); -} - -TAsyncStatus TConsoleClient::RemoveVolatileConfig( - const TString& cluster, - ui64 version, - const TVector<ui64>& ids, - const TClusterConfigSettings& settings) { - return Impl_->RemoveVolatileConfig(cluster, version, ids, settings); -} - -TAsyncGetConfigResult TConsoleClient::GetConfig(const TClusterConfigSettings& settings) { - return Impl_->GetConfig(settings); -} - -TAsyncResolveConfigResult TConsoleClient::ResolveConfig( - const TString& config, - const TMap<ui64, TString>& volatileConfigs, - const TMap<TString, TString>& labels, - const TClusterConfigSettings& settings) { - return Impl_->ResolveConfig(config, volatileConfigs, labels, settings); -} - -TAsyncResolveConfigResult TConsoleClient::ResolveConfig( - const TString& config, - const TMap<ui64, TString>& volatileConfigs, - const TClusterConfigSettings& settings) { - return Impl_->ResolveConfig(config, volatileConfigs, settings); -} - -} // namespace NYdb::NConsole diff --git a/ydb/public/sdk/cpp/client/draft/ydb_console.h b/ydb/public/sdk/cpp/client/draft/ydb_console.h deleted file mode 100644 index d5903c8321..0000000000 --- a/ydb/public/sdk/cpp/client/draft/ydb_console.h +++ /dev/null @@ -1,124 +0,0 @@ -#pragma once - -#include <ydb/public/api/grpc/draft/ydb_console_v1.grpc.pb.h> -#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> -#include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> -#include <ydb/public/sdk/cpp/client/ydb_common_client/settings.h> -#include <ydb/public/sdk/cpp/client/ydb_types/request_settings.h> -#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -#include <util/generic/string.h> -#include <util/generic/map.h> -#include <util/system/types.h> - -#include <memory> - -namespace NYdb::NConsole { - -struct TGetConfigResult : public TStatus { - TGetConfigResult( - TStatus status, - TString clusterName, - ui64 version, - TString config, - TMap<ui64, TString> volatileConfigs) - : TStatus(std::move(status)) - , ClusterName_(std::move(clusterName)) - , Version_(version) - , Config_(std::move(config)) - , VolatileConfigs_(std::move(volatileConfigs)) - {} - - const TString& GetClusterName() const { - return ClusterName_; - } - - ui64 GetVersion() const { - return Version_; - } - - const TString& GetConfig() const { - return Config_; - } - - const TMap<ui64, TString>& GetVolatileConfigs() const { - return VolatileConfigs_; - } - -private: - TString ClusterName_; - ui64 Version_; - TString Config_; - TMap<ui64, TString> VolatileConfigs_; -}; - -using TAsyncGetConfigResult = NThreading::TFuture<TGetConfigResult>; - -struct TResolveConfigResult : public TStatus { - TResolveConfigResult( - TStatus status, - TString config) - : TStatus(std::move(status)) - , Config_(std::move(config)) - {} - - const TString& GetConfig() const { - return Config_; - } - -private: - TString Config_; -}; - -using TAsyncResolveConfigResult = NThreading::TFuture<TResolveConfigResult>; - -struct TConsoleClientSettings : public TCommonClientSettingsBase<TConsoleClientSettings> { - using TSelf = TConsoleClientSettings; -}; - -struct TClusterConfigSettings : public NYdb::TOperationRequestSettings<TClusterConfigSettings> {}; - -class TConsoleClient { -public: - class TImpl; - - explicit TConsoleClient(const TDriver& driver); - - // Apply config - TAsyncStatus ApplyConfig(const TString& config, const TClusterConfigSettings& settings = {}); - - // Add volatile config - TAsyncStatus AddVolatileConfig( - const TString& config, - ui64 id, ui64 version, - const TString& cluster, - const TClusterConfigSettings& settings = {}); - - // Remove specific volatile config or all of them - TAsyncStatus RemoveVolatileConfig( - const TString& cluster, - ui64 version, - const TVector<ui64>& ids, - const TClusterConfigSettings& settings = {}); - - // Get current cluster configs - TAsyncGetConfigResult GetConfig(const TClusterConfigSettings& settings = {}); - - // Resolve arbitrary config for specific labels - TAsyncResolveConfigResult ResolveConfig( - const TString& config, - const TMap<ui64, TString>& volatileConfigs, - const TMap<TString, TString>& labels, - const TClusterConfigSettings& settings = {}); - - // Resolve arbitrary config for all label combinations - TAsyncResolveConfigResult ResolveConfig( - const TString& config, - const TMap<ui64, TString>& volatileConfigs, - const TClusterConfigSettings& settings = {}); - -private: - std::shared_ptr<TImpl> Impl_; -}; - -} // namespace NYdb::NConsole diff --git a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp new file mode 100644 index 0000000000..76d5cbc9b5 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.cpp @@ -0,0 +1,429 @@ +#include "ydb_dynamic_config.h" + +#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> +#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> + +namespace NYdb::NDynamicConfig { + +class TDynamicConfigClient::TImpl : public TClientImplCommon<TDynamicConfigClient::TImpl> { +public: + TImpl(std::shared_ptr<TGRpcConnectionsImpl> connections) + : TClientImplCommon(std::move(connections), TDynamicConfigClientSettings{}) + { + } + + TAsyncStatus SetConfig(const TString& config, bool dryRun, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::SetConfigRequest>(settings); + request.set_config(config); + request.set_dry_run(dryRun); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::SetConfigRequest, Ydb::DynamicConfig::SetConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncSetConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus ReplaceConfig(const TString& config, bool dryRun, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::ReplaceConfigRequest>(settings); + request.set_config(config); + request.set_dry_run(dryRun); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::ReplaceConfigRequest, Ydb::DynamicConfig::ReplaceConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncReplaceConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus DropConfig(const TString& cluster, ui64 version, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::DropConfigRequest>(settings); + + request.mutable_identity()->set_cluster(cluster); + request.mutable_identity()->set_version(version); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::DropConfigRequest, Ydb::DynamicConfig::DropConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncDropConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus AddVolatileConfig(const TString& config, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::AddVolatileConfigRequest>(settings); + request.set_config(config); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::AddVolatileConfigRequest, Ydb::DynamicConfig::AddVolatileConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncAddVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus RemoveVolatileConfig(const TString& cluster, ui64 version, const TVector<ui64>& ids, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::RemoveVolatileConfigRequest>(settings); + + request.mutable_identity()->set_cluster(cluster); + request.mutable_identity()->set_version(version); + + for (auto& id: ids) { + request.mutable_ids()->add_ids(id); + } + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::RemoveVolatileConfigRequest, Ydb::DynamicConfig::RemoveVolatileConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus RemoveAllVolatileConfigs(const TString& cluster, ui64 version, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::RemoveVolatileConfigRequest>(settings); + + request.mutable_identity()->set_cluster(cluster); + request.mutable_identity()->set_version(version); + request.set_all(true); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::RemoveVolatileConfigRequest, Ydb::DynamicConfig::RemoveVolatileConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus ForceRemoveVolatileConfig(const TVector<ui64>& ids, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::RemoveVolatileConfigRequest>(settings); + + for (auto& id: ids) { + request.mutable_ids()->add_ids(id); + } + + request.set_force(true); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::RemoveVolatileConfigRequest, Ydb::DynamicConfig::RemoveVolatileConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + TAsyncStatus ForceRemoveAllVolatileConfigs(const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::RemoveVolatileConfigRequest>(settings); + + request.set_all(true); + request.set_force(true); + + return RunSimple<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::RemoveVolatileConfigRequest, Ydb::DynamicConfig::RemoveVolatileConfigResponse>( + std::move(request), + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncRemoveVolatileConfig, + TRpcRequestSettings::Make(settings)); + } + + + TAsyncGetNodeLabelsResult GetNodeLabels(ui64 nodeId, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::GetNodeLabelsRequest>(settings); + request.set_node_id(nodeId); + + auto promise = NThreading::NewPromise<TGetNodeLabelsResult>(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + TMap<TString, TString> labels; + if (Ydb::DynamicConfig::GetNodeLabelsResult result; any && any->UnpackTo(&result)) { + for (auto& label : result.labels()) { + labels[label.label()] = label.value(); + } + } + + TGetNodeLabelsResult val(TStatus(std::move(status)), std::move(labels)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::GetNodeLabelsRequest, Ydb::DynamicConfig::GetNodeLabelsResponse>( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncGetNodeLabels, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncGetConfigResult GetConfig(const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::GetConfigRequest>(settings); + + auto promise = NThreading::NewPromise<TGetConfigResult>(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + TString clusterName = "<unknown>"; + ui64 version = 0; + TString config; + TMap<ui64, TString> volatileConfigs; + if (Ydb::DynamicConfig::GetConfigResult result; any && any->UnpackTo(&result)) { + clusterName = result.identity().cluster(); + version = result.identity().version(); + config = result.config(); + for (const auto& config : result.volatile_configs()) { + volatileConfigs.emplace(config.id(), config.config()); + } + } + + TGetConfigResult val(TStatus(std::move(status)), std::move(clusterName), version, std::move(config), std::move(volatileConfigs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::GetConfigRequest, Ydb::DynamicConfig::GetConfigResponse>( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncGetConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncGetMetadataResult GetMetadata(const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::GetMetadataRequest>(settings); + + auto promise = NThreading::NewPromise<TGetMetadataResult>(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + TString metadata; + TMap<ui64, TString> volatileConfigs; + if (Ydb::DynamicConfig::GetMetadataResult result; any && any->UnpackTo(&result)) { + metadata = result.metadata(); + for (const auto& config : result.volatile_configs()) { + volatileConfigs.emplace(config.id(), config.metadata()); + } + } + + TGetMetadataResult val(TStatus(std::move(status)),std::move(metadata), std::move(volatileConfigs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::GetMetadataRequest, Ydb::DynamicConfig::GetMetadataResponse>( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncGetMetadata, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncResolveConfigResult ResolveConfig(const TString& config, const TMap<ui64, TString>& volatileConfigs, const TMap<TString, TString>& labels, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::ResolveConfigRequest>(settings); + request.set_config(config); + for (auto& [id, volatileConfig] : volatileConfigs) { + auto* proto = request.add_volatile_configs(); + proto->set_id(id); + proto->set_config(volatileConfig); + } + for (auto& [name, value] : labels) { + auto* proto = request.add_labels(); + proto->set_label(name); + proto->set_value(value); + } + + auto promise = NThreading::NewPromise<TResolveConfigResult>(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + TString config; + if (Ydb::DynamicConfig::ResolveConfigResult result; any && any->UnpackTo(&result)) { + config = result.config(); + } + + TResolveConfigResult val(TStatus(std::move(status)), std::move(config)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::ResolveConfigRequest, Ydb::DynamicConfig::ResolveConfigResponse>( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncResolveConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncResolveConfigResult ResolveConfig(const TString& config, const TMap<ui64, TString>& volatileConfigs, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::ResolveAllConfigRequest>(settings); + request.set_config(config); + for (auto& [id, volatileConfig] : volatileConfigs) { + auto* proto = request.add_volatile_configs(); + proto->set_id(id); + proto->set_config(volatileConfig); + } + + auto promise = NThreading::NewPromise<TResolveConfigResult>(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + TString config; + if (Ydb::DynamicConfig::ResolveAllConfigResult result; any && any->UnpackTo(&result)) { + config = result.config(); + } + + TResolveConfigResult val(TStatus(std::move(status)), std::move(config)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::ResolveAllConfigRequest, Ydb::DynamicConfig::ResolveAllConfigResponse>( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncResolveAllConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncVerboseResolveConfigResult VerboseResolveConfig(const TString& config, const TMap<ui64, TString>& volatileConfigs, const TClusterConfigSettings& settings = {}) { + auto request = MakeOperationRequest<Ydb::DynamicConfig::ResolveAllConfigRequest>(settings); + request.set_config(config); + for (auto& [id, volatileConfig] : volatileConfigs) { + auto* proto = request.add_volatile_configs(); + proto->set_id(id); + proto->set_config(volatileConfig); + } + request.set_verbose_response(true); + + auto promise = NThreading::NewPromise<TVerboseResolveConfigResult>(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + auto convert = [] (const Ydb::DynamicConfig::YamlLabelExt::LabelType& label) -> TVerboseResolveConfigResult::TLabel::EType { + switch(label) { + case Ydb::DynamicConfig::YamlLabelExt::NOT_SET: + return TVerboseResolveConfigResult::TLabel::EType::Negative; + case Ydb::DynamicConfig::YamlLabelExt::COMMON: + return TVerboseResolveConfigResult::TLabel::EType::Common; + case Ydb::DynamicConfig::YamlLabelExt::EMPTY: + return TVerboseResolveConfigResult::TLabel::EType::Empty; + default: + Y_FAIL("unexpected enum value"); + } + }; + + TSet<TString> labels; + TVerboseResolveConfigResult::ConfigByLabelSet configs; + + if (Ydb::DynamicConfig::ResolveAllConfigResult result; any && any->UnpackTo(&result)) { + for (auto& config : result.configs()) { + TSet<TVector<TVerboseResolveConfigResult::TLabel>> labelSets; + for (auto& labelSet : config.label_sets()) { + TVector<TVerboseResolveConfigResult::TLabel> set; + for (auto& label : labelSet.labels()) { + labels.insert(label.label()); + set.push_back(TVerboseResolveConfigResult::TLabel{convert(label.type()), label.value()}); + } + labelSets.insert(set); + } + configs[labelSets] = config.config(); + } + } + + TVerboseResolveConfigResult val(TStatus(std::move(status)), std::move(labels), std::move(configs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::DynamicConfig::V1::DynamicConfigService, Ydb::DynamicConfig::ResolveAllConfigRequest, Ydb::DynamicConfig::ResolveAllConfigResponse>( + std::move(request), + extractor, + &Ydb::DynamicConfig::V1::DynamicConfigService::Stub::AsyncResolveAllConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } +}; + +TDynamicConfigClient::TDynamicConfigClient(const TDriver& driver) + : Impl_(new TDynamicConfigClient::TImpl(CreateInternalInterface(driver))) +{} + +TAsyncStatus TDynamicConfigClient::SetConfig( + const TString& config, + bool dryRun, + const TClusterConfigSettings& settings) { + return Impl_->SetConfig(config, dryRun, settings); +} + +TAsyncStatus TDynamicConfigClient::ReplaceConfig( + const TString& config, + bool dryRun, + const TClusterConfigSettings& settings) { + return Impl_->ReplaceConfig(config, dryRun, settings); +} + +TAsyncStatus TDynamicConfigClient::DropConfig( + const TString& cluster, + ui64 version, + const TClusterConfigSettings& settings) { + return Impl_->DropConfig(cluster, version, settings); +} + +TAsyncStatus TDynamicConfigClient::AddVolatileConfig( + const TString& config, + const TClusterConfigSettings& settings) { + return Impl_->AddVolatileConfig(config, settings); +} + +TAsyncStatus TDynamicConfigClient::RemoveVolatileConfig( + const TString& cluster, + ui64 version, + const TVector<ui64>& ids, + const TClusterConfigSettings& settings) { + return Impl_->RemoveVolatileConfig(cluster, version, ids, settings); +} + +TAsyncStatus TDynamicConfigClient::RemoveAllVolatileConfigs( + const TString& cluster, + ui64 version, + const TClusterConfigSettings& settings) { + return Impl_->RemoveAllVolatileConfigs(cluster, version, settings); +} + +TAsyncStatus TDynamicConfigClient::ForceRemoveVolatileConfig( + const TVector<ui64>& ids, + const TClusterConfigSettings& settings) { + return Impl_->ForceRemoveVolatileConfig(ids, settings); +} + +TAsyncStatus TDynamicConfigClient::ForceRemoveAllVolatileConfigs( + const TClusterConfigSettings& settings) { + return Impl_->ForceRemoveAllVolatileConfigs(settings); +} + +TAsyncGetMetadataResult TDynamicConfigClient::GetMetadata(const TClusterConfigSettings& settings) { + return Impl_->GetMetadata(settings); +} + +TAsyncGetConfigResult TDynamicConfigClient::GetConfig(const TClusterConfigSettings& settings) { + return Impl_->GetConfig(settings); +} + +TAsyncGetNodeLabelsResult TDynamicConfigClient::GetNodeLabels(ui64 nodeId, const TClusterConfigSettings& settings) { + return Impl_->GetNodeLabels(nodeId, settings); +} + +TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( + const TString& config, + const TMap<ui64, TString>& volatileConfigs, + const TMap<TString, TString>& labels, + const TClusterConfigSettings& settings) { + return Impl_->ResolveConfig(config, volatileConfigs, labels, settings); +} + +TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( + const TString& config, + const TMap<ui64, TString>& volatileConfigs, + const TClusterConfigSettings& settings) { + return Impl_->ResolveConfig(config, volatileConfigs, settings); +} + +TAsyncVerboseResolveConfigResult TDynamicConfigClient::VerboseResolveConfig( + const TString& config, + const TMap<ui64, TString>& volatileConfigs, + const TClusterConfigSettings& settings) { + return Impl_->VerboseResolveConfig(config, volatileConfigs, settings); +} + +} // namespace NYdb::NDynamicConfig diff --git a/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h new file mode 100644 index 0000000000..1925ae2f5d --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ydb_dynamic_config.h @@ -0,0 +1,253 @@ +#pragma once + +#include <ydb/public/api/grpc/draft/ydb_dynamic_config_v1.grpc.pb.h> +#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> +#include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> +#include <ydb/public/sdk/cpp/client/ydb_common_client/settings.h> +#include <ydb/public/sdk/cpp/client/ydb_types/request_settings.h> +#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> + +#include <util/generic/string.h> +#include <util/generic/set.h> +#include <util/generic/map.h> +#include <util/system/types.h> + +#include <memory> + +namespace NYdb::NDynamicConfig { + +struct TGetConfigResult : public TStatus { + TGetConfigResult( + TStatus&& status, + TString&& clusterName, + ui64 version, + TString&& config, + TMap<ui64, TString>&& volatileConfigs) + : TStatus(std::move(status)) + , ClusterName_(std::move(clusterName)) + , Version_(version) + , Config_(std::move(config)) + , VolatileConfigs_(std::move(volatileConfigs)) + {} + + const TString& GetClusterName() const { + return ClusterName_; + } + + ui64 GetVersion() const { + return Version_; + } + + const TString& GetConfig() const { + return Config_; + } + + const TMap<ui64, TString>& GetVolatileConfigs() const { + return VolatileConfigs_; + } + +private: + TString ClusterName_; + ui64 Version_; + TString Config_; + TMap<ui64, TString> VolatileConfigs_; +}; + +using TAsyncGetConfigResult = NThreading::TFuture<TGetConfigResult>; + +struct TGetMetadataResult : public TStatus { + TGetMetadataResult( + TStatus&& status, + TString&& metadata, + TMap<ui64, TString>&& volatileConfigs) + : TStatus(std::move(status)) + , Metadata_(std::move(metadata)) + , VolatileConfigs_(std::move(volatileConfigs)) + {} + + const TString& GetMetadata() const { + return Metadata_; + } + + const TMap<ui64, TString>& GetVolatileConfigs() const { + return VolatileConfigs_; + } + +private: + TString Metadata_; + TMap<ui64, TString> VolatileConfigs_; +}; + +using TAsyncGetMetadataResult = NThreading::TFuture<TGetMetadataResult>; + +struct TResolveConfigResult : public TStatus { + TResolveConfigResult( + TStatus&& status, + TString&& config) + : TStatus(std::move(status)) + , Config_(std::move(config)) + {} + + const TString& GetConfig() const { + return Config_; + } + +private: + TString Config_; +}; + +using TAsyncResolveConfigResult = NThreading::TFuture<TResolveConfigResult>; + +struct TGetNodeLabelsResult : public TStatus { + TGetNodeLabelsResult( + TStatus&& status, + TMap<TString, TString>&& labels) + : TStatus(std::move(status)) + , Labels_(std::move(labels)) + {} + + const TMap<TString, TString>& GetLabels() const { + return Labels_; + } + +private: + TMap<TString, TString> Labels_; +}; + +using TAsyncGetNodeLabelsResult = NThreading::TFuture<TGetNodeLabelsResult>; + +struct TVerboseResolveConfigResult : public TStatus { + struct TLabel { + enum class EType { + Negative = 0, + Empty, + Common, + }; + + EType Type; + TString Value; + + bool operator<(const TLabel& other) const { + int lhs = static_cast<int>(Type); + int rhs = static_cast<int>(other.Type); + return std::tie(lhs, Value) < std::tie(rhs, other.Value); + } + + bool operator==(const TLabel& other) const { + int lhs = static_cast<int>(Type); + int rhs = static_cast<int>(other.Type); + return std::tie(lhs, Value) == std::tie(rhs, other.Value); + } + }; + + using ConfigByLabelSet = TMap<TSet<TVector<TLabel>>, TString>; + + TVerboseResolveConfigResult( + TStatus&& status, + TSet<TString>&& labels, + ConfigByLabelSet&& configs) + : TStatus(std::move(status)) + , Labels_(std::move(labels)) + , Configs_(std::move(configs)) + {} + + const TSet<TString>& GetLabels() const { + return Labels_; + } + + const ConfigByLabelSet& GetConfigs() const { + return Configs_; + } + +private: + TSet<TString> Labels_; + ConfigByLabelSet Configs_; +}; + +using TAsyncVerboseResolveConfigResult = NThreading::TFuture<TVerboseResolveConfigResult>; + + +struct TDynamicConfigClientSettings : public TCommonClientSettingsBase<TDynamicConfigClientSettings> { + using TSelf = TDynamicConfigClientSettings; +}; + +struct TClusterConfigSettings : public NYdb::TOperationRequestSettings<TClusterConfigSettings> {}; + +class TDynamicConfigClient { +public: + class TImpl; + + explicit TDynamicConfigClient(const TDriver& driver); + + // Set config + TAsyncStatus SetConfig(const TString& config, bool dryRun = false, const TClusterConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfig(const TString& config, bool dryRun = false, const TClusterConfigSettings& settings = {}); + + // Drop config + TAsyncStatus DropConfig( + const TString& cluster, + ui64 version, + const TClusterConfigSettings& settings = {}); + + // Add volatile config + TAsyncStatus AddVolatileConfig( + const TString& config, + const TClusterConfigSettings& settings = {}); + + // Remove specific volatile configs + TAsyncStatus RemoveVolatileConfig( + const TString& cluster, + ui64 version, + const TVector<ui64>& ids, + const TClusterConfigSettings& settings = {}); + + // Remove all volatile config + TAsyncStatus RemoveAllVolatileConfigs( + const TString& cluster, + ui64 version, + const TClusterConfigSettings& settings = {}); + + // Remove specific volatile configs + TAsyncStatus ForceRemoveVolatileConfig( + const TVector<ui64>& ids, + const TClusterConfigSettings& settings = {}); + + // Remove all volatile config + TAsyncStatus ForceRemoveAllVolatileConfigs( + const TClusterConfigSettings& settings = {}); + + // Get current cluster configs metadata + TAsyncGetMetadataResult GetMetadata(const TClusterConfigSettings& settings = {}); + + // Get current cluster configs + TAsyncGetConfigResult GetConfig(const TClusterConfigSettings& settings = {}); + + // Get current cluster configs + TAsyncGetNodeLabelsResult GetNodeLabels(ui64 nodeId, const TClusterConfigSettings& settings = {}); + + // Resolve arbitrary config for specific labels + TAsyncResolveConfigResult ResolveConfig( + const TString& config, + const TMap<ui64, TString>& volatileConfigs, + const TMap<TString, TString>& labels, + const TClusterConfigSettings& settings = {}); + + // Resolve arbitrary config for all label combinations + TAsyncResolveConfigResult ResolveConfig( + const TString& config, + const TMap<ui64, TString>& volatileConfigs, + const TClusterConfigSettings& settings = {}); + + // Resolve arbitrary config for all label combinations + TAsyncVerboseResolveConfigResult VerboseResolveConfig( + const TString& config, + const TMap<ui64, TString>& volatileConfigs, + const TClusterConfigSettings& settings = {}); + +private: + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NYdb::NDynamicConfig diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 01c75f3cc5..8a636f30a8 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -301,6 +301,11 @@ class TTableDescription::TImpl { break; } + // tiering + if (proto.tiering().size()) { + Tiering_ = proto.tiering(); + } + // column families ColumnFamilies_.reserve(proto.column_families_size()); for (const auto& family : proto.column_families()) { @@ -517,6 +522,10 @@ public: return TtlSettings_; } + const TMaybe<TString>& GetTiering() const { + return Tiering_; + } + const TString& GetOwner() const { return Owner_; } @@ -593,6 +602,7 @@ private: TVector<TIndexDescription> Indexes_; TVector<TChangefeedDescription> Changefeeds_; TMaybe<TTtlSettings> TtlSettings_; + TMaybe<TString> Tiering_; TString Owner_; TVector<NScheme::TPermissions> Permissions_; TVector<NScheme::TPermissions> EffectivePermissions_; @@ -657,6 +667,10 @@ TMaybe<TTtlSettings> TTableDescription::GetTtlSettings() const { return Impl_->GetTtlSettings(); } +TMaybe<TString> TTableDescription::GetTiering() const { + return Impl_->GetTiering(); +} + const TString& TTableDescription::GetOwner() const { return Impl_->GetOwner(); } @@ -837,6 +851,10 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con ttl->SerializeTo(*request.mutable_ttl_settings()); } + if (const auto& tiering = Impl_->GetTiering()) { + request.set_tiering(*tiering); + } + if (Impl_->HasStorageSettings()) { request.mutable_storage_settings()->CopyFrom(Impl_->GetStorageSettings().GetProto()); } @@ -4412,6 +4430,26 @@ TChangefeedDescription& TChangefeedDescription::WithInitialScan() { return *this; } +TChangefeedDescription& TChangefeedDescription::AddAttribute(const TString& key, const TString& value) { + Attributes_[key] = value; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::SetAttributes(const THashMap<TString, TString>& attrs) { + Attributes_ = attrs; + return *this; +} + +TChangefeedDescription& TChangefeedDescription::SetAttributes(THashMap<TString, TString>&& attrs) { + Attributes_ = std::move(attrs); + return *this; +} + +TChangefeedDescription& TChangefeedDescription::WithAwsRegion(const TString& value) { + AwsRegion_ = value; + return *this; +} + const TString& TChangefeedDescription::GetName() const { return Name_; } @@ -4436,6 +4474,14 @@ bool TChangefeedDescription::GetInitialScan() const { return InitialScan_; } +const THashMap<TString, TString>& TChangefeedDescription::GetAttributes() const { + return Attributes_; +} + +const TString& TChangefeedDescription::GetAwsRegion() const { + return AwsRegion_; +} + template <typename TProto> TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { EChangefeedMode mode; @@ -4465,6 +4511,9 @@ TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { case Ydb::Table::ChangefeedFormat::FORMAT_JSON: format = EChangefeedFormat::Json; break; + case Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON: + format = EChangefeedFormat::DynamoDBStreamsJson; + break; default: format = EChangefeedFormat::Unknown; break; @@ -4474,6 +4523,9 @@ TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { if (proto.virtual_timestamps()) { ret.WithVirtualTimestamps(); } + if (!proto.aws_region().empty()) { + ret.WithAwsRegion(proto.aws_region()); + } if constexpr (std::is_same_v<TProto, Ydb::Table::ChangefeedDescription>) { switch (proto.state()) { @@ -4492,6 +4544,10 @@ TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { } } + for (const auto& [key, value] : proto.attributes()) { + ret.Attributes_[key] = value; + } + return ret; } @@ -4499,6 +4555,7 @@ void TChangefeedDescription::SerializeTo(Ydb::Table::Changefeed& proto) const { proto.set_name(Name_); proto.set_virtual_timestamps(VirtualTimestamps_); proto.set_initial_scan(InitialScan_); + proto.set_aws_region(AwsRegion_); switch (Mode_) { case EChangefeedMode::KeysOnly: @@ -4524,6 +4581,9 @@ void TChangefeedDescription::SerializeTo(Ydb::Table::Changefeed& proto) const { case EChangefeedFormat::Json: proto.set_format(Ydb::Table::ChangefeedFormat::FORMAT_JSON); break; + case EChangefeedFormat::DynamoDBStreamsJson: + proto.set_format(Ydb::Table::ChangefeedFormat::FORMAT_DYNAMODB_STREAMS_JSON); + break; case EChangefeedFormat::Unknown: break; } @@ -4533,6 +4593,10 @@ void TChangefeedDescription::SerializeTo(Ydb::Table::Changefeed& proto) const { retention.set_seconds(RetentionPeriod_->Seconds()); retention.set_nanos(RetentionPeriod_->NanoSecondsOfSecond()); } + + for (const auto& [key, value] : Attributes_) { + (*proto.mutable_attributes())[key] = value; + } } TString TChangefeedDescription::ToString() const { @@ -4552,6 +4616,10 @@ void TChangefeedDescription::Out(IOutputStream& o) const { o << ", retention_period: " << *RetentionPeriod_; } + if (AwsRegion_) { + o << ", aws_region: " << AwsRegion_; + } + o << " }"; } @@ -4559,7 +4627,8 @@ bool operator==(const TChangefeedDescription& lhs, const TChangefeedDescription& return lhs.GetName() == rhs.GetName() && lhs.GetMode() == rhs.GetMode() && lhs.GetFormat() == rhs.GetFormat() - && lhs.GetVirtualTimestamps() == rhs.GetVirtualTimestamps(); + && lhs.GetVirtualTimestamps() == rhs.GetVirtualTimestamps() + && lhs.GetAwsRegion() == rhs.GetAwsRegion(); } bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs) { diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index a13cde5856..ce50fc0057 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -207,6 +207,12 @@ public: TChangefeedDescription& WithRetentionPeriod(const TDuration& value); // Initial scan will output the current state of the table first TChangefeedDescription& WithInitialScan(); + // Attributes + TChangefeedDescription& AddAttribute(const TString& key, const TString& value); + TChangefeedDescription& SetAttributes(const THashMap<TString, TString>& attrs); + TChangefeedDescription& SetAttributes(THashMap<TString, TString>&& attrs); + // Value that will be emitted in the `awsRegion` field of the record in DocumentTableJson format + TChangefeedDescription& WithAwsRegion(const TString& value); const TString& GetName() const; EChangefeedMode GetMode() const; @@ -214,6 +220,8 @@ public: EChangefeedState GetState() const; bool GetVirtualTimestamps() const; bool GetInitialScan() const; + const THashMap<TString, TString>& GetAttributes() const; + const TString& GetAwsRegion() const; void SerializeTo(Ydb::Table::Changefeed& proto) const; TString ToString() const; @@ -234,6 +242,8 @@ private: bool VirtualTimestamps_ = false; std::optional<TDuration> RetentionPeriod_; bool InitialScan_ = false; + THashMap<TString, TString> Attributes_; + TString AwsRegion_; }; bool operator==(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs); @@ -449,6 +459,7 @@ public: TVector<TIndexDescription> GetIndexDescriptions() const; TVector<TChangefeedDescription> GetChangefeedDescriptions() const; TMaybe<TTtlSettings> GetTtlSettings() const; + TMaybe<TString> GetTiering() const; // Deprecated. Use GetEntry() of TDescribeTableResult instead const TString& GetOwner() const; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h index 7a96e160cb..516acd4934 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table_enum.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table_enum.h @@ -41,6 +41,7 @@ enum class EChangefeedMode { enum class EChangefeedFormat { Json /* "JSON" */, + DynamoDBStreamsJson /* "DYNAMODB_STREAMS_JSON" */, Unknown = std::numeric_limits<int>::max() }; diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index 71c69959c6..e341650179 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -552,6 +552,10 @@ void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) { out << "Void"sv; break; + case TTypeParser::ETypeKind::Null: + out << "Null"sv; + break; + default: ThrowFatalError(TStringBuilder() << "Unexpected type kind: " << parser.GetKind()); @@ -989,6 +993,11 @@ bool TPgValue::IsText() const { //////////////////////////////////////////////////////////////////////////////// +TUuidValue::TUuidValue(ui64 low_128, ui64 high_128) { + Buf_.Halfs[0] = low_128; + Buf_.Halfs[1] = high_128; +} + TUuidValue::TUuidValue(const Ydb::Value& valueProto) { Buf_.Halfs[0] = valueProto.low_128(); Buf_.Halfs[1] = valueProto.high_128(); diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index 5c7c885daa..d37f63bc45 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -244,6 +244,7 @@ struct TPgValue { struct TUuidValue { TString ToString() const; + TUuidValue(ui64 low_128, ui64 high_128); TUuidValue(const Ydb::Value& uuidValueProto); TUuidValue(const TString& uuidString); diff --git a/ydb/services/CMakeLists.txt b/ydb/services/CMakeLists.txt index e4f15a917e..03be2fc52a 100644 --- a/ydb/services/CMakeLists.txt +++ b/ydb/services/CMakeLists.txt @@ -9,9 +9,9 @@ add_subdirectory(auth) add_subdirectory(bg_tasks) add_subdirectory(cms) -add_subdirectory(console) add_subdirectory(datastreams) add_subdirectory(discovery) +add_subdirectory(dynamic_config) add_subdirectory(fq) add_subdirectory(kesus) add_subdirectory(lib) diff --git a/ydb/services/console/CMakeLists.darwin.txt b/ydb/services/dynamic_config/CMakeLists.darwin.txt index c190afd4bc..045e49cf2b 100644 --- a/ydb/services/console/CMakeLists.darwin.txt +++ b/ydb/services/dynamic_config/CMakeLists.darwin.txt @@ -8,8 +8,8 @@ add_subdirectory(ut) -add_library(ydb-services-console) -target_link_libraries(ydb-services-console PUBLIC +add_library(ydb-services-dynamic_config) +target_link_libraries(ydb-services-dynamic_config PUBLIC contrib-libs-cxxsupp yutil cpp-grpc-server @@ -20,6 +20,6 @@ target_link_libraries(ydb-services-console PUBLIC public-lib-operation_id cpp-client-resources ) -target_sources(ydb-services-console PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console/grpc_service.cpp +target_sources(ydb-services-dynamic_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config/grpc_service.cpp ) diff --git a/ydb/services/console/CMakeLists.linux-aarch64.txt b/ydb/services/dynamic_config/CMakeLists.linux-aarch64.txt index a119a985c7..d9fcccd7a2 100644 --- a/ydb/services/console/CMakeLists.linux-aarch64.txt +++ b/ydb/services/dynamic_config/CMakeLists.linux-aarch64.txt @@ -8,8 +8,8 @@ add_subdirectory(ut) -add_library(ydb-services-console) -target_link_libraries(ydb-services-console PUBLIC +add_library(ydb-services-dynamic_config) +target_link_libraries(ydb-services-dynamic_config PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -21,6 +21,6 @@ target_link_libraries(ydb-services-console PUBLIC public-lib-operation_id cpp-client-resources ) -target_sources(ydb-services-console PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console/grpc_service.cpp +target_sources(ydb-services-dynamic_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config/grpc_service.cpp ) diff --git a/ydb/services/console/CMakeLists.linux.txt b/ydb/services/dynamic_config/CMakeLists.linux.txt index a119a985c7..d9fcccd7a2 100644 --- a/ydb/services/console/CMakeLists.linux.txt +++ b/ydb/services/dynamic_config/CMakeLists.linux.txt @@ -8,8 +8,8 @@ add_subdirectory(ut) -add_library(ydb-services-console) -target_link_libraries(ydb-services-console PUBLIC +add_library(ydb-services-dynamic_config) +target_link_libraries(ydb-services-dynamic_config PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -21,6 +21,6 @@ target_link_libraries(ydb-services-console PUBLIC public-lib-operation_id cpp-client-resources ) -target_sources(ydb-services-console PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console/grpc_service.cpp +target_sources(ydb-services-dynamic_config PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config/grpc_service.cpp ) diff --git a/ydb/services/dynamic_config/CMakeLists.txt b/ydb/services/dynamic_config/CMakeLists.txt new file mode 100644 index 0000000000..3e0811fb22 --- /dev/null +++ b/ydb/services/dynamic_config/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (APPLE) + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/services/console/console_ut.cpp b/ydb/services/dynamic_config/dynamic_config_ut.cpp index 9a11d9f461..9a11d9f461 100644 --- a/ydb/services/console/console_ut.cpp +++ b/ydb/services/dynamic_config/dynamic_config_ut.cpp diff --git a/ydb/services/console/grpc_service.cpp b/ydb/services/dynamic_config/grpc_service.cpp index c77c9d745e..2cd003d8e5 100644 --- a/ydb/services/console/grpc_service.cpp +++ b/ydb/services/dynamic_config/grpc_service.cpp @@ -1,13 +1,13 @@ #include "grpc_service.h" -#include <ydb/core/grpc_services/service_console.h> +#include <ydb/core/grpc_services/service_dynamic_config.h> #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/base/base.h> namespace NKikimr { namespace NGRpcService { -void TGRpcConsoleService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { +void TGRpcDynamicConfigService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); using namespace Ydb; @@ -15,20 +15,24 @@ void TGRpcConsoleService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { #error ADD_REQUEST macro already defined #endif #define ADD_REQUEST(NAME, CB) \ - MakeIntrusive<TGRpcRequest<Console::NAME##Request, Console::NAME##Response, TGRpcConsoleService>> \ + MakeIntrusive<TGRpcRequest<DynamicConfig::NAME##Request, DynamicConfig::NAME##Response, TGRpcDynamicConfigService>> \ (this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ActorSystem_->Send(GRpcRequestProxyId_, \ - new TGrpcRequestOperationCall<Console::NAME##Request, Console::NAME##Response> \ + new TGrpcRequestOperationCall<DynamicConfig::NAME##Request, DynamicConfig::NAME##Response> \ (ctx, &CB, TRequestAuxSettings{RLSWITCH(TRateLimiterMode::Rps), nullptr})); \ - }, &Console::V1::ConsoleService::AsyncService::Request ## NAME, \ + }, &DynamicConfig::V1::DynamicConfigService::AsyncService::Request ## NAME, \ #NAME, logger, getCounterBlock("console", #NAME))->Run(); - ADD_REQUEST(ApplyConfig, DoApplyConfigRequest) + ADD_REQUEST(SetConfig, DoSetConfigRequest) + ADD_REQUEST(ReplaceConfig, DoReplaceConfigRequest) + ADD_REQUEST(DropConfig, DoDropConfigRequest) ADD_REQUEST(AddVolatileConfig, DoAddVolatileConfigRequest) ADD_REQUEST(RemoveVolatileConfig, DoRemoveVolatileConfigRequest) ADD_REQUEST(GetConfig, DoGetConfigRequest) + ADD_REQUEST(GetMetadata, DoGetMetadataRequest) + ADD_REQUEST(GetNodeLabels, DoGetNodeLabelsRequest) ADD_REQUEST(ResolveConfig, DoResolveConfigRequest) ADD_REQUEST(ResolveAllConfig, DoResolveAllConfigRequest) diff --git a/ydb/services/console/grpc_service.h b/ydb/services/dynamic_config/grpc_service.h index 46820c894e..dc2617f5d2 100644 --- a/ydb/services/console/grpc_service.h +++ b/ydb/services/dynamic_config/grpc_service.h @@ -2,17 +2,17 @@ #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/grpc/server/grpc_server.h> -#include <ydb/public/api/grpc/draft/ydb_console_v1.grpc.pb.h> +#include <ydb/public/api/grpc/draft/ydb_dynamic_config_v1.grpc.pb.h> #include <ydb/core/grpc_services/base/base_service.h> namespace NKikimr { namespace NGRpcService { -class TGRpcConsoleService - : public TGrpcServiceBase<Ydb::Console::V1::ConsoleService> +class TGRpcDynamicConfigService + : public TGrpcServiceBase<Ydb::DynamicConfig::V1::DynamicConfigService> { public: - using TGrpcServiceBase<Ydb::Console::V1::ConsoleService>::TGrpcServiceBase; + using TGrpcServiceBase<Ydb::DynamicConfig::V1::DynamicConfigService>::TGrpcServiceBase; private: void SetupIncomingRequests(NGrpc::TLoggerPtr logger); }; diff --git a/ydb/services/console/ut/CMakeLists.darwin.txt b/ydb/services/dynamic_config/ut/CMakeLists.darwin.txt index 07c1143199..3db0ffe72b 100644 --- a/ydb/services/console/ut/CMakeLists.darwin.txt +++ b/ydb/services/dynamic_config/ut/CMakeLists.darwin.txt @@ -7,19 +7,19 @@ -add_executable(ydb-services-console-ut) -target_compile_options(ydb-services-console-ut PRIVATE +add_executable(ydb-services-dynamic_config-ut) +target_compile_options(ydb-services-dynamic_config-ut PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-services-console-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console +target_include_directories(ydb-services-dynamic_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config ) -target_link_libraries(ydb-services-console-ut PUBLIC +target_link_libraries(ydb-services-dynamic_config-ut PUBLIC contrib-libs-cxxsupp yutil library-cpp-cpuid_check cpp-testing-unittest_main - ydb-services-console + ydb-services-dynamic_config library-cpp-getopt cpp-grpc-client cpp-regex-pcre @@ -27,7 +27,7 @@ target_link_libraries(ydb-services-console-ut PUBLIC core-testlib-default ydb-services-cms ) -target_link_options(ydb-services-console-ut PRIVATE +target_link_options(ydb-services-dynamic_config-ut PRIVATE -Wl,-no_deduplicate -Wl,-sdk_version,10.15 -fPIC @@ -35,18 +35,18 @@ target_link_options(ydb-services-console-ut PRIVATE -framework CoreFoundation ) -target_sources(ydb-services-console-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console/console_ut.cpp +target_sources(ydb-services-dynamic_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config/dynamic_config_ut.cpp ) add_test( NAME - ydb-services-console-ut + ydb-services-dynamic_config-ut COMMAND - ydb-services-console-ut + ydb-services-dynamic_config-ut --print-before-suite --print-before-test --fork-tests --print-times --show-fails ) -vcs_info(ydb-services-console-ut) +vcs_info(ydb-services-dynamic_config-ut) diff --git a/ydb/services/console/ut/CMakeLists.linux-aarch64.txt b/ydb/services/dynamic_config/ut/CMakeLists.linux-aarch64.txt index f16d31d583..170d188833 100644 --- a/ydb/services/console/ut/CMakeLists.linux-aarch64.txt +++ b/ydb/services/dynamic_config/ut/CMakeLists.linux-aarch64.txt @@ -7,20 +7,20 @@ -add_executable(ydb-services-console-ut) -target_compile_options(ydb-services-console-ut PRIVATE +add_executable(ydb-services-dynamic_config-ut) +target_compile_options(ydb-services-dynamic_config-ut PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-services-console-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console +target_include_directories(ydb-services-dynamic_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config ) -target_link_libraries(ydb-services-console-ut PUBLIC +target_link_libraries(ydb-services-dynamic_config-ut PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil library-cpp-lfalloc cpp-testing-unittest_main - ydb-services-console + ydb-services-dynamic_config library-cpp-getopt cpp-grpc-client cpp-regex-pcre @@ -28,7 +28,7 @@ target_link_libraries(ydb-services-console-ut PUBLIC core-testlib-default ydb-services-cms ) -target_link_options(ydb-services-console-ut PRIVATE +target_link_options(ydb-services-dynamic_config-ut PRIVATE -ldl -lrt -Wl,--no-as-needed @@ -38,18 +38,18 @@ target_link_options(ydb-services-console-ut PRIVATE -lrt -ldl ) -target_sources(ydb-services-console-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console/console_ut.cpp +target_sources(ydb-services-dynamic_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config/dynamic_config_ut.cpp ) add_test( NAME - ydb-services-console-ut + ydb-services-dynamic_config-ut COMMAND - ydb-services-console-ut + ydb-services-dynamic_config-ut --print-before-suite --print-before-test --fork-tests --print-times --show-fails ) -vcs_info(ydb-services-console-ut) +vcs_info(ydb-services-dynamic_config-ut) diff --git a/ydb/services/console/ut/CMakeLists.linux.txt b/ydb/services/dynamic_config/ut/CMakeLists.linux.txt index 9f24fd2412..e28eeacaab 100644 --- a/ydb/services/console/ut/CMakeLists.linux.txt +++ b/ydb/services/dynamic_config/ut/CMakeLists.linux.txt @@ -7,14 +7,14 @@ -add_executable(ydb-services-console-ut) -target_compile_options(ydb-services-console-ut PRIVATE +add_executable(ydb-services-dynamic_config-ut) +target_compile_options(ydb-services-dynamic_config-ut PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-services-console-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console +target_include_directories(ydb-services-dynamic_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config ) -target_link_libraries(ydb-services-console-ut PUBLIC +target_link_libraries(ydb-services-dynamic_config-ut PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -22,7 +22,7 @@ target_link_libraries(ydb-services-console-ut PUBLIC libs-tcmalloc-no_percpu_cache library-cpp-cpuid_check cpp-testing-unittest_main - ydb-services-console + ydb-services-dynamic_config library-cpp-getopt cpp-grpc-client cpp-regex-pcre @@ -30,7 +30,7 @@ target_link_libraries(ydb-services-console-ut PUBLIC core-testlib-default ydb-services-cms ) -target_link_options(ydb-services-console-ut PRIVATE +target_link_options(ydb-services-dynamic_config-ut PRIVATE -ldl -lrt -Wl,--no-as-needed @@ -40,18 +40,18 @@ target_link_options(ydb-services-console-ut PRIVATE -lrt -ldl ) -target_sources(ydb-services-console-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/services/console/console_ut.cpp +target_sources(ydb-services-dynamic_config-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/dynamic_config/dynamic_config_ut.cpp ) add_test( NAME - ydb-services-console-ut + ydb-services-dynamic_config-ut COMMAND - ydb-services-console-ut + ydb-services-dynamic_config-ut --print-before-suite --print-before-test --fork-tests --print-times --show-fails ) -vcs_info(ydb-services-console-ut) +vcs_info(ydb-services-dynamic_config-ut) diff --git a/ydb/services/dynamic_config/ut/CMakeLists.txt b/ydb/services/dynamic_config/ut/CMakeLists.txt new file mode 100644 index 0000000000..3e0811fb22 --- /dev/null +++ b/ydb/services/dynamic_config/ut/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (APPLE) + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/services/persqueue_v1/actors/helpers.cpp b/ydb/services/persqueue_v1/actors/helpers.cpp index d4db92ba16..d957753686 100644 --- a/ydb/services/persqueue_v1/actors/helpers.cpp +++ b/ydb/services/persqueue_v1/actors/helpers.cpp @@ -36,10 +36,10 @@ TMaybe<ui32> GetPartitionFromConfigOptions( TMaybe<ui32> ret; if (preferred < Max<ui32>()) { ret = preferred; - } else if (!useRoundRobin){ - ret = encodedSrcId.Hash % partPerTablet; } else if (firstClass) { ret = NKikimr::NDataStreams::V1::CalculateShardFromSrcId(encodedSrcId.OriginalSourceId, partPerTablet); + } else if (!useRoundRobin){ + ret = encodedSrcId.Hash % partPerTablet; } return ret; } diff --git a/ydb/services/persqueue_v1/actors/schema_actors.cpp b/ydb/services/persqueue_v1/actors/schema_actors.cpp index f9bdd72464..06b9d22c5e 100644 --- a/ydb/services/persqueue_v1/actors/schema_actors.cpp +++ b/ydb/services/persqueue_v1/actors/schema_actors.cpp @@ -549,11 +549,10 @@ void TDescribeTopicActorImpl::RequestTablet(TTabletInfo& tablet, const TActorCon tablet.Pipe = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tablet.TabletId, NTabletPipe::TClientConfig(NTabletPipe::TClientRetryPolicy::WithRetries()))); if (tablet.TabletId == BalancerTabletId) { - THolder<NKikimr::TEvPersQueue::TEvGetReadSessionsInfo> ev(new NKikimr::TEvPersQueue::TEvGetReadSessionsInfo(Consumer)); + THolder<NKikimr::TEvPersQueue::TEvGetReadSessionsInfo> ev(new NKikimr::TEvPersQueue::TEvGetReadSessionsInfo(NPersQueue::ConvertNewConsumerName(Consumer, ctx))); NTabletPipe::SendData(ctx, tablet.Pipe, ev.Release()); - } else { - THolder<NKikimr::TEvPersQueue::TEvStatus> ev(new NKikimr::TEvPersQueue::TEvStatus(Consumer.empty() ? "" : NPersQueue::ConvertNewConsumerName(Consumer), Consumer.empty())); + THolder<NKikimr::TEvPersQueue::TEvStatus> ev(new NKikimr::TEvPersQueue::TEvStatus(Consumer.empty() ? "" : NPersQueue::ConvertNewConsumerName(Consumer, ctx), Consumer.empty())); NTabletPipe::SendData(ctx, tablet.Pipe, ev.Release()); } ++RequestsInfly; diff --git a/ydb/services/persqueue_v1/actors/write_session_actor.ipp b/ydb/services/persqueue_v1/actors/write_session_actor.ipp index e563fb5241..72e81135cc 100644 --- a/ydb/services/persqueue_v1/actors/write_session_actor.ipp +++ b/ydb/services/persqueue_v1/actors/write_session_actor.ipp @@ -1627,8 +1627,9 @@ void TWriteSessionActor<UseMigrationProtocol>::RecheckACL(const TActorContext& c RequestNotChecked = false; InitCheckSchema(ctx); } + const auto& pqConfig = AppData(ctx)->PQConfig; // ToDo[migration] - separate flag for having config tables - if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen() + if ((!pqConfig.GetTopicsAreFirstClassCitizen() || pqConfig.GetUseSrcIdMetaMappingInFirstClass()) && !SourceIdUpdatesInflight && ctx.Now() - LastSourceIdUpdate > SOURCEID_UPDATE_PERIOD ) { diff --git a/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp b/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp index 7d15e63eb8..40d7f89561 100644 --- a/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp +++ b/ydb/services/persqueue_v1/first_class_src_ids_ut.cpp @@ -2,6 +2,7 @@ #include <ydb/public/sdk/cpp/client/ydb_topic/topic.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h> +#include <ydb/services/persqueue_v1/actors/helpers.h> namespace NKikimr::NPersQueueTests { using namespace Tests; @@ -79,12 +80,43 @@ Y_UNIT_TEST_SUITE(TFstClassSrcIdPQTest) { UNIT_ASSERT(res); writer->Close(); }; - Y_UNUSED(alterAndCheck); + alterAndCheck(2); alterAndCheck(4); alterAndCheck(12); // ydbDriver = nullptr; } + + Y_UNIT_TEST(ProperPartitionSelected) { + TString topic = "/Root/topic-f3"; + auto [server, ydbDriver] = Setup("/Root/otherTopic", false); + //auto& server_ = server; + auto& driver = ydbDriver; + + ui32 partCount = 15; + TString srcId = "mySrcID"; + server->AnnoyingClient->CreateTopicNoLegacy(topic, partCount); + + auto pqClient = NYdb::NTopic::TTopicClient(*driver); + auto topicSettings = NYdb::NTopic::TAlterTopicSettings(); + topicSettings.BeginAddConsumer("debug"); + auto alterRes = pqClient.AlterTopic(topic, topicSettings).GetValueSync(); + UNIT_ASSERT(alterRes.IsSuccess()); + + auto partExpected = NKikimr::NDataStreams::V1::CalculateShardFromSrcId(srcId, partCount); + Y_VERIFY(partExpected < partCount); + auto writer = CreateSimpleWriter(*driver, topic, srcId); + auto res = writer->Write("test-data", writer->GetInitSeqNo() + 1); + UNIT_ASSERT(res); + writer->Close(); + + NYdb::NPersQueue::TReadSessionSettings readerSettings; + readerSettings.ConsumerName("debug").AppendTopics(topic); + auto reader = CreateReader(*driver, readerSettings); + auto mbEv = GetNextMessageSkipAssignment(reader); + UNIT_ASSERT(mbEv.Defined()); + UNIT_ASSERT_VALUES_EQUAL(mbEv->GetPartitionStream()->GetPartitionGroupId(), partExpected + 1); + } } } // namespace NKikimr::NPersQueueTests diff --git a/ydb/services/ydb/ydb_olapstore_ut.cpp b/ydb/services/ydb/ydb_olapstore_ut.cpp index 2fdced9f4a..f7b7144b60 100644 --- a/ydb/services/ydb/ydb_olapstore_ut.cpp +++ b/ydb/services/ydb/ydb_olapstore_ut.cpp @@ -780,10 +780,6 @@ Y_UNIT_TEST_SUITE(YdbOlapStore) { } Y_UNIT_TEST_TWIN(LogTsRangeDescending, NotNull) { - if (NotNull) { - // Wait for fix https://st.yandex-team.ru/KIKIMR-16920 - return; - } TString query(R"( --PRAGMA AnsiInForEmptyOrNullableItemsCollections; diff --git a/ydb/tests/functional/cms_config_cache/main.py b/ydb/tests/functional/cms_config_cache/main.py index 8df6cd3ad2..ec624b9a8c 100644 --- a/ydb/tests/functional/cms_config_cache/main.py +++ b/ydb/tests/functional/cms_config_cache/main.py @@ -46,6 +46,11 @@ LogConfig { } """ + log_entry = """Entry { + Component: "FLAT_TX_SCHEMESHARD" + Level: 7 + }""" + @classmethod def setup_class(cls): configurator = KikimrConfigGenerator() @@ -69,7 +74,7 @@ LogConfig { def test_cache_log_settings(self): with open(self._cms_config_cache_file_path(), 'r') as file: - assert 'LogConfig' not in file.read(), "initial node config should not contain LogConfig items" + assert self.log_entry not in file.read(), "initial node config should not contain LogConfig items" self.cluster.client.add_config_item(self.sample_log_config) timeout = 60 step = 1 @@ -79,7 +84,7 @@ LogConfig { time.sleep(step) cur += step with open(self._cms_config_cache_file_path(), 'r') as file: - config_updated = 'LogConfig' in file.read() + config_updated = self.log_entry in file.read() assert config_updated, "log config wasn't updated" def test_cms_config_cache(self): diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema index 00cd34dd26..b568ce8044 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema @@ -724,6 +724,16 @@ "ColumnId": 12, "ColumnName": "LastSeenReady", "ColumnType": "Uint64" + }, + { + "ColumnId": 13, + "ColumnName": "LastGotReplicating", + "ColumnType": "Uint64" + }, + { + "ColumnId": 14, + "ColumnName": "ReplicationTime", + "ColumnType": "Uint64" } ], "ColumnsDropped": [], @@ -741,7 +751,9 @@ 9, 10, 11, - 12 + 12, + 13, + 14 ], "RoomID": 0, "Codec": 0, diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index a53dde2a59..108f66893c 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -341,6 +341,11 @@ "ColumnId": 9, "ColumnName": "IsBackup", "ColumnType": "Bool" + }, + { + "ColumnId": 10, + "ColumnName": "ReplicationConfig", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -355,7 +360,8 @@ 6, 7, 8, - 9 + 9, + 10 ], "RoomID": 0, "Codec": 0, @@ -1235,6 +1241,11 @@ ], "ColumnsAdded": [ { + "ColumnId": 29, + "ColumnName": "ImportsLimit", + "ColumnType": "Uint64" + }, + { "ColumnId": 1, "ColumnName": "PathId", "ColumnType": "Uint64" @@ -1368,12 +1379,18 @@ "ColumnId": 27, "ColumnName": "TableCdcStreamsLimit", "ColumnType": "Uint64" + }, + { + "ColumnId": 28, + "ColumnName": "ExportsLimit", + "ColumnType": "Uint64" } ], "ColumnsDropped": [], "ColumnFamilies": { "0": { "Columns": [ + 29, 1, 2, 3, @@ -1400,7 +1417,8 @@ 24, 25, 26, - 27 + 27, + 28 ], "RoomID": 0, "Codec": 0, @@ -3450,6 +3468,11 @@ "ColumnId": 10, "ColumnName": "IsBackup", "ColumnType": "Bool" + }, + { + "ColumnId": 11, + "ColumnName": "ReplicationConfig", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -3465,7 +3488,8 @@ 7, 8, 9, - 10 + 10, + 11 ], "RoomID": 0, "Codec": 0, @@ -6104,11 +6128,6 @@ ], "ColumnsAdded": [ { - "ColumnId": 7, - "ColumnName": "VirtualTimestamps", - "ColumnType": "Bool" - }, - { "ColumnId": 1, "ColumnName": "OwnerPathId", "ColumnType": "Uint64" @@ -6137,19 +6156,30 @@ "ColumnId": 6, "ColumnName": "Format", "ColumnType": "Uint32" + }, + { + "ColumnId": 7, + "ColumnName": "VirtualTimestamps", + "ColumnType": "Bool" + }, + { + "ColumnId": 8, + "ColumnName": "AwsRegion", + "ColumnType": "Utf8" } ], "ColumnsDropped": [], "ColumnFamilies": { "0": { "Columns": [ - 7, 1, 2, 3, 4, 5, - 6 + 6, + 7, + 8 ], "RoomID": 0, "Codec": 0, @@ -6176,11 +6206,6 @@ ], "ColumnsAdded": [ { - "ColumnId": 7, - "ColumnName": "VirtualTimestamps", - "ColumnType": "Bool" - }, - { "ColumnId": 1, "ColumnName": "OwnerPathId", "ColumnType": "Uint64" @@ -6209,19 +6234,30 @@ "ColumnId": 6, "ColumnName": "Format", "ColumnType": "Uint32" + }, + { + "ColumnId": 7, + "ColumnName": "VirtualTimestamps", + "ColumnType": "Bool" + }, + { + "ColumnId": 8, + "ColumnName": "AwsRegion", + "ColumnType": "Utf8" } ], "ColumnsDropped": [], "ColumnFamilies": { "0": { "Columns": [ - 7, 1, 2, 3, 4, 5, - 6 + 6, + 7, + 8 ], "RoomID": 0, "Codec": 0, diff --git a/ydb/tests/functional/sqs/common/test_account_actions.py b/ydb/tests/functional/sqs/common/test_account_actions.py index 539192febb..81aa591f9a 100644 --- a/ydb/tests/functional/sqs/common/test_account_actions.py +++ b/ydb/tests/functional/sqs/common/test_account_actions.py @@ -5,14 +5,18 @@ import pytest from hamcrest import assert_that, not_none, has_item, is_not from ydb.tests.library.sqs.test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation -from ydb.tests.library.sqs.test_base import TABLES_FORMAT_PARAMS +from ydb.tests.library.sqs.test_base import TABLES_FORMAT_PARAMS, HAS_QUEUES_PARAMS class AccountActionsTest(KikimrSqsTestBase): @pytest.mark.parametrize(**TABLES_FORMAT_PARAMS) - def test_manage_account(self, tables_format): + @pytest.mark.parametrize(**HAS_QUEUES_PARAMS) + def test_manage_account(self, tables_format, has_queues): self._init_with_params(tables_format=tables_format) + user_name = 'pupkin' + self._sqs_api = self._create_api_for_user(user_name) + create_user_result = self._sqs_api.create_user(user_name) assert_that( create_user_result, @@ -23,6 +27,10 @@ class AccountActionsTest(KikimrSqsTestBase): user_list, has_item(user_name) ) + if has_queues: + self._sqs_api.create_queue('queue.fifo', is_fifo=True) + self._sqs_api.create_queue('queue', is_fifo=False) + delete_user_result = self._sqs_api.delete_user(user_name) assert_that( delete_user_result, diff --git a/ydb/tests/functional/sqs/common/test_acl.py b/ydb/tests/functional/sqs/common/test_acl.py index 76ea8abffa..3c1212159a 100644 --- a/ydb/tests/functional/sqs/common/test_acl.py +++ b/ydb/tests/functional/sqs/common/test_acl.py @@ -140,7 +140,7 @@ class SqsACLTest(KikimrSqsTestBase): result = self._sqs_api.send_message(_queue_url, data) assert_that(result, result_predicate) return result - except: + except Exception: if retries == 0: raise time.sleep(0.1) @@ -164,9 +164,9 @@ class SqsACLTest(KikimrSqsTestBase): __send_message_with_retries(queue_url, 'superdata', is_(none())) # no access again. that's a pity result = self._sqs_api.list_permissions(self._username) - assert('Account' in str(result)) - assert(berkanavt_sid in str(result)) - assert('Permissions' in str(result)) + assert 'Account' in str(result) + assert berkanavt_sid in str(result) + assert 'Permissions' in str(result) class SqsWithForceAuthorizationTest(KikimrSqsTestBase): diff --git a/ydb/tests/functional/sqs/common/test_format_without_version.py b/ydb/tests/functional/sqs/common/test_format_without_version.py index 9b93f9fced..9795783ecb 100644 --- a/ydb/tests/functional/sqs/common/test_format_without_version.py +++ b/ydb/tests/functional/sqs/common/test_format_without_version.py @@ -23,7 +23,7 @@ class QueueWithoutVersionTest(KikimrSqsTestBase): if info.TabletId not in self.used_tablets: self.used_tablets.append(info.TabletId) return info.TabletId - assert(False) + assert False def get_table_path(self, table=None): table_path = f'{self.sqs_root}/{self._username}/{self.queue_name}' diff --git a/ydb/tests/functional/sqs/common/test_multiplexing_tables_format.py b/ydb/tests/functional/sqs/common/test_multiplexing_tables_format.py index 05479f69a7..0a8c5cf036 100644 --- a/ydb/tests/functional/sqs/common/test_multiplexing_tables_format.py +++ b/ydb/tests/functional/sqs/common/test_multiplexing_tables_format.py @@ -20,7 +20,7 @@ class MultiplexingTablesFormatTest(KikimrSqsTestBase): self.create_queue(is_fifo) except yatest.common.process.ExecutionError: return - assert(False) + assert False def create_queue_with_wrong_tables_format(self, tables_format): self._set_tables_format(tables_format='qwerty') diff --git a/ydb/tests/functional/sqs/common/test_queue_attributes_validation.py b/ydb/tests/functional/sqs/common/test_queue_attributes_validation.py index e09dbbfd38..c4f7b3430d 100644 --- a/ydb/tests/functional/sqs/common/test_queue_attributes_validation.py +++ b/ydb/tests/functional/sqs/common/test_queue_attributes_validation.py @@ -30,7 +30,7 @@ class TestQueueAttributesInCompatibilityMode(KikimrSqsTestBase): assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024')) try: queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': 'troll'}, retries=1) - except: + except Exception: pass else: raise Exception('only parseable attributes are valid') @@ -56,13 +56,13 @@ class TestQueueAttributesValidation(KikimrSqsTestBase): val = int(attributes[attr]) # change attributes slightly attributes_to_set_and_check[attr] = str(val - 1 if val > 0 else val + 1) - assert(attributes_to_set_and_check[attr] != attributes[attr]) + assert attributes_to_set_and_check[attr] != attributes[attr] content_based_dedup_key = 'ContentBasedDeduplication' if is_fifo: val = attributes[content_based_dedup_key] attributes_to_set_and_check[content_based_dedup_key] = 'true' if val == 'false' else 'false' - assert(attributes_to_set_and_check[content_based_dedup_key] != attributes[content_based_dedup_key]) + assert attributes_to_set_and_check[content_based_dedup_key] != attributes[content_based_dedup_key] self._sqs_api.set_queue_attributes(queue_url, attributes_to_set_and_check) # match set attributes @@ -75,23 +75,23 @@ class TestQueueAttributesValidation(KikimrSqsTestBase): try: self._sqs_api.set_queue_attributes(queue_url, {attr: '2.5'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert(attr in str(e)) + assert 'InvalidAttributeValue' in str(e) + assert attr in str(e) else: assert_that(False) # expected InvalidAttributeValue exception try: self._sqs_api.set_queue_attributes(queue_url, {attr: '2147483647'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert(attr in str(e)) + assert 'InvalidAttributeValue' in str(e) + assert attr in str(e) else: assert_that(False) # expected InvalidAttributeValue exception try: self._sqs_api.set_queue_attributes(queue_url, {'trololo': 'ololo'}) except Exception as e: - assert('InvalidAttributeName' in str(e)) + assert 'InvalidAttributeName' in str(e) else: assert_that(False) # expected InvalidAttributeName exception @@ -101,8 +101,8 @@ class TestQueueAttributesValidation(KikimrSqsTestBase): try: self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'omg'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert('FifoQueue' in str(e)) + assert 'InvalidAttributeValue' in str(e) + assert 'FifoQueue' in str(e) else: assert_that(False) # expected InvalidAttributeValue exception @@ -110,15 +110,15 @@ class TestQueueAttributesValidation(KikimrSqsTestBase): try: self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'}) except Exception as e: - assert('InvalidAttributeValue' in str(e)) - assert('Modifying queue type is not supported' in str(e)) + assert 'InvalidAttributeValue' in str(e) + assert 'Modifying queue type is not supported' in str(e) else: assert_that(False) # expected InvalidAttributeValue exception else: try: self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'}) except Exception as e: - assert('InvalidAttributeName' in str(e)) + assert 'InvalidAttributeName' in str(e) else: assert_that(False) # expected InvalidAttributeName exception @@ -176,13 +176,13 @@ class TestQueueAttributesValidation(KikimrSqsTestBase): try: queue_url = self._sqs_api.create_queue('new_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2147483647'}) except Exception as e: - assert(attr in str(e)) + assert attr in str(e) else: assert_that(False) # expected some exception try: queue_url = self._sqs_api.create_queue('new2_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2.5'}) except Exception as e: - assert(attr in str(e)) + assert attr in str(e) else: assert_that(False) # expected some exception diff --git a/ydb/tests/library/sqs/test_base.py b/ydb/tests/library/sqs/test_base.py index 138ca4608a..83af71dba2 100644 --- a/ydb/tests/library/sqs/test_base.py +++ b/ydb/tests/library/sqs/test_base.py @@ -61,6 +61,12 @@ VISIBILITY_CHANGE_METHOD_PARAMS = { 'ids': ['with_delete_message', 'with_change_visibility'], } +HAS_QUEUES_PARAMS = { + 'argnames': 'has_queues', + 'argvalues': [True, False], + 'ids': ['with_queues', 'without_queues'] +} + def get_sqs_client_path(): return yatest_common.binary_path("ydb/core/ymq/client/bin/sqs") |