diff options
author | uzhas <uzhas@ydb.tech> | 2023-01-27 18:41:26 +0300 |
---|---|---|
committer | uzhas <uzhas@ydb.tech> | 2023-01-27 18:41:26 +0300 |
commit | e82ace91ad0eb705a39c4542d5c8a84a999a7120 (patch) | |
tree | af46f1061af2fd2a1e24a34fc4029a135dbd11f6 | |
parent | 9e21b8ae23c8cbb1c19a4012ff9f7ac07bdb6e1b (diff) | |
download | ydb-e82ace91ad0eb705a39c4542d5c8a84a999a7120.tar.gz |
use fq scope in core/yq
21 files changed, 63 insertions, 50 deletions
diff --git a/ydb/core/yq/libs/actors/CMakeLists.darwin.txt b/ydb/core/yq/libs/actors/CMakeLists.darwin.txt index c8b64cb56a8..1400276e968 100644 --- a/ydb/core/yq/libs/actors/CMakeLists.darwin.txt +++ b/ydb/core/yq/libs/actors/CMakeLists.darwin.txt @@ -55,7 +55,7 @@ target_link_libraries(yq-libs-actors PUBLIC yql-sql-settings yql-utils-actor_log api-protos - public-lib-yq + public-lib-fq cpp-client-ydb_table providers-clickhouse-provider providers-common-codec diff --git a/ydb/core/yq/libs/actors/CMakeLists.linux-aarch64.txt b/ydb/core/yq/libs/actors/CMakeLists.linux-aarch64.txt index d729eb3d120..0b587a07f8d 100644 --- a/ydb/core/yq/libs/actors/CMakeLists.linux-aarch64.txt +++ b/ydb/core/yq/libs/actors/CMakeLists.linux-aarch64.txt @@ -56,7 +56,7 @@ target_link_libraries(yq-libs-actors PUBLIC yql-sql-settings yql-utils-actor_log api-protos - public-lib-yq + public-lib-fq cpp-client-ydb_table providers-clickhouse-provider providers-common-codec diff --git a/ydb/core/yq/libs/actors/CMakeLists.linux.txt b/ydb/core/yq/libs/actors/CMakeLists.linux.txt index d729eb3d120..0b587a07f8d 100644 --- a/ydb/core/yq/libs/actors/CMakeLists.linux.txt +++ b/ydb/core/yq/libs/actors/CMakeLists.linux.txt @@ -56,7 +56,7 @@ target_link_libraries(yq-libs-actors PUBLIC yql-sql-settings yql-utils-actor_log api-protos - public-lib-yq + public-lib-fq cpp-client-ydb_table providers-clickhouse-provider providers-common-codec diff --git a/ydb/core/yq/libs/actors/pending_fetcher.cpp b/ydb/core/yq/libs/actors/pending_fetcher.cpp index f447db19582..a3d411ffec9 100644 --- a/ydb/core/yq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/yq/libs/actors/pending_fetcher.cpp @@ -1,5 +1,3 @@ -#include <ydb/core/yq/libs/config/protos/pinger.pb.h> -#include <ydb/core/yq/libs/config/protos/fq_config.pb.h> #include "proxy.h" #include "nodes_manager.h" @@ -43,11 +41,13 @@ #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> +#include <ydb/public/lib/fq/scope.h> #include <ydb/core/yq/libs/common/compression.h> #include <ydb/core/yq/libs/common/entity_id.h> #include <ydb/core/yq/libs/events/events.h> - +#include <ydb/core/yq/libs/config/protos/fq_config.pb.h> +#include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/private_client/internal_service.h> @@ -316,9 +316,7 @@ private: NDq::SetYqlLogLevels(NActors::NLog::PRI_TRACE); - const TVector<TString> path = StringSplitter(task.scope()).Split('/').SkipEmpty(); // yandexcloud://{folder_id} - const TString folderId = path.size() == 2 && path.front().StartsWith(NYdb::NYq::TScope::YandexCloudScopeSchema) - ? path.back() : TString{}; + const TString folderId = NYdb::NFq::TScope(task.scope()).ParseFolder(); const TString cloudId = task.sensor_labels().at("cloud_id"); const TString queryId = task.query_id().value(); diff --git a/ydb/core/yq/libs/actors/pinger.cpp b/ydb/core/yq/libs/actors/pinger.cpp index de3eddbaa72..bbce638faa9 100644 --- a/ydb/core/yq/libs/actors/pinger.cpp +++ b/ydb/core/yq/libs/actors/pinger.cpp @@ -1,18 +1,20 @@ -#include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include "proxy.h" -#include <util/datetime/base.h> +#include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/private_client/internal_service.h> +#include <ydb/public/lib/fq/scope.h> + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/core/log.h> #include <library/cpp/protobuf/interop/cast.h> +#include <util/datetime/base.h> #include <util/generic/utility.h> #include <deque> @@ -138,7 +140,7 @@ class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { public: TPingerActor( const TString& tenantName, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& userId, const TString& id, const TString& ownerId, @@ -415,7 +417,7 @@ private: TConfig Config; const TString TenantName; - const TScope Scope; + const NYdb::NFq::TScope Scope; const TString UserId; const TString Id; const TString OwnerId; @@ -439,7 +441,7 @@ private: IActor* CreatePingerActor( const TString& tenantName, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& userId, const TString& id, const TString& ownerId, diff --git a/ydb/core/yq/libs/actors/proxy.h b/ydb/core/yq/libs/actors/proxy.h index 9dd4c9ba764..da481f1b87a 100644 --- a/ydb/core/yq/libs/actors/proxy.h +++ b/ydb/core/yq/libs/actors/proxy.h @@ -1,10 +1,9 @@ #pragma once -#include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include "run_actor_params.h" -#include <util/datetime/base.h> #include <ydb/core/mon/mon.h> +#include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/private_client/private_client.h> #include <ydb/core/yq/libs/shared_resources/db_pool.h> @@ -16,13 +15,16 @@ #include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h> #include <ydb/library/yql/providers/dq/actors/proto_builder.h> #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h> +#include <ydb/library/yql/providers/common/metrics/service_counters.h> #include <ydb/library/yql/providers/pq/cm_client/client.h> +#include <ydb/public/lib/fq/scope.h> + #include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/time_provider/time_provider.h> #include <library/cpp/random_provider/random_provider.h> +#include <library/cpp/time_provider/time_provider.h> -#include <ydb/library/yql/providers/common/metrics/service_counters.h> +#include <util/datetime/base.h> namespace NKikimr { namespace NMiniKQL { @@ -81,7 +83,7 @@ NActors::IActor* CreateResultWriter( NActors::IActor* CreatePingerActor( const TString& tenantName, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& userId, const TString& id, const TString& owner, @@ -95,14 +97,14 @@ NActors::IActor* CreateRateLimiterResourceCreator( const NActors::TActorId& parent, const TString& ownerId, const TString& queryId, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& tenant); NActors::IActor* CreateRateLimiterResourceDeleter( const NActors::TActorId& parent, const TString& ownerId, const TString& queryId, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& tenant); TString MakeInternalError(const TString& text); diff --git a/ydb/core/yq/libs/actors/rate_limiter.cpp b/ydb/core/yq/libs/actors/rate_limiter.cpp index 9f8c819ce25..dfcde9fac52 100644 --- a/ydb/core/yq/libs/actors/rate_limiter.cpp +++ b/ydb/core/yq/libs/actors/rate_limiter.cpp @@ -1,6 +1,7 @@ #include "proxy.h" #include <ydb/core/yq/libs/private_client/events.h> +#include <ydb/public/lib/fq/scope.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> @@ -108,7 +109,7 @@ NActors::IActor* CreateRateLimiterResourceCreator( const NActors::TActorId& parent, const TString& ownerId, const TString& queryId, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& tenant) { Fq::Private::CreateRateLimiterResourceRequest req; @@ -123,7 +124,7 @@ NActors::IActor* CreateRateLimiterResourceDeleter( const NActors::TActorId& parent, const TString& ownerId, const TString& queryId, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& tenant) { Fq::Private::DeleteRateLimiterResourceRequest req; diff --git a/ydb/core/yq/libs/actors/run_actor_params.cpp b/ydb/core/yq/libs/actors/run_actor_params.cpp index a25b03fa3b2..fea230bbc3d 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.cpp +++ b/ydb/core/yq/libs/actors/run_actor_params.cpp @@ -21,7 +21,7 @@ TRunActorParams::TRunActorParams( const ::NYq::NConfig::TPingerConfig& pingerConfig, const ::NYq::NConfig::TRateLimiterConfig& rateLimiterConfig, const TString& sql, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& authToken, const TActorId& databaseResolver, const TString& queryId, diff --git a/ydb/core/yq/libs/actors/run_actor_params.h b/ydb/core/yq/libs/actors/run_actor_params.h index 21f23270fcc..8717a2a5e1b 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.h +++ b/ydb/core/yq/libs/actors/run_actor_params.h @@ -12,6 +12,8 @@ #include <ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h> #include <ydb/library/yql/providers/pq/cm_client/client.h> +#include <ydb/public/lib/fq/scope.h> + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/time_provider/time_provider.h> #include <library/cpp/random_provider/random_provider.h> @@ -36,7 +38,7 @@ struct TRunActorParams { // TODO2 : Change name const ::NYq::NConfig::TPingerConfig& pingerConfig, const ::NYq::NConfig::TRateLimiterConfig& rateLimiterConfig, const TString& sql, - const TScope& scope, + const NYdb::NFq::TScope& scope, const TString& authToken, const NActors::TActorId& databaseResolver, const TString& queryId, @@ -93,7 +95,7 @@ struct TRunActorParams { // TODO2 : Change name const ::NYq::NConfig::TPingerConfig PingerConfig; const ::NYq::NConfig::TRateLimiterConfig RateLimiterConfig; const TString Sql; - const TScope Scope; + const NYdb::NFq::TScope Scope; const TString AuthToken; const NActors::TActorId DatabaseResolver; const TString QueryId; diff --git a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.darwin.txt b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.darwin.txt index 9f0f917231c..663f18cb5cc 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.darwin.txt +++ b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.darwin.txt @@ -28,11 +28,12 @@ target_link_libraries(libs-control_plane_storage-internal PUBLIC libs-quota_manager-events libs-rate_limiter-events yq-libs-ydb + ydb-library-protobuf_printer ydb-library-security + yql-public-issue + public-lib-fq cpp-client-ydb_scheme cpp-client-ydb_value - ydb-library-protobuf_printer - yql-public-issue ) target_sources(libs-control_plane_storage-internal PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp diff --git a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux-aarch64.txt b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux-aarch64.txt index 5b69f6913bd..861c5ef59ce 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux-aarch64.txt +++ b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux-aarch64.txt @@ -29,11 +29,12 @@ target_link_libraries(libs-control_plane_storage-internal PUBLIC libs-quota_manager-events libs-rate_limiter-events yq-libs-ydb + ydb-library-protobuf_printer ydb-library-security + yql-public-issue + public-lib-fq cpp-client-ydb_scheme cpp-client-ydb_value - ydb-library-protobuf_printer - yql-public-issue ) target_sources(libs-control_plane_storage-internal PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp diff --git a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux.txt b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux.txt index 5b69f6913bd..861c5ef59ce 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux.txt +++ b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.linux.txt @@ -29,11 +29,12 @@ target_link_libraries(libs-control_plane_storage-internal PUBLIC libs-quota_manager-events libs-rate_limiter-events yq-libs-ydb + ydb-library-protobuf_printer ydb-library-security + yql-public-issue + public-lib-fq cpp-client-ydb_scheme cpp-client-ydb_value - ydb-library-protobuf_printer - yql-public-issue ) target_sources(libs-control_plane_storage-internal PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp diff --git a/ydb/core/yq/libs/control_plane_storage/internal/rate_limiter_resources.cpp b/ydb/core/yq/libs/control_plane_storage/internal/rate_limiter_resources.cpp index 5eadd427835..b4a0b20c094 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/rate_limiter_resources.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/rate_limiter_resources.cpp @@ -1,6 +1,6 @@ #include "utils.h" -#include <ydb/public/lib/yq/scope.h> +#include <ydb/public/lib/fq/scope.h> #include <ydb/core/yq/libs/control_plane_storage/request_actor.h> #include <ydb/core/yq/libs/db_schema/db_schema.h> #include <ydb/core/yq/libs/quota_manager/quota_manager.h> @@ -141,10 +141,7 @@ public: } void ParseScope(const TString& scope) { - const TVector<TString> path = StringSplitter(scope).Split('/').SkipEmpty(); // yandexcloud://{folder_id} - if (path.size() == 2 && path.front().StartsWith(NYdb::NYq::TScope::YandexCloudScopeSchema)) { - FolderId = path.back(); - } + FolderId = NYdb::NFq::TScope(scope).ParseFolder(); } protected: diff --git a/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp b/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp index 3f02a179601..009572f792a 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp @@ -4,9 +4,12 @@ #include <ydb/core/metering/bill_record.h> #include <ydb/core/metering/metering.h> +#include <ydb/public/lib/fq/scope.h> namespace NYq { +using NYdb::NFq::TScope; + NYql::TIssues ValidateWriteResultData(const TString& resultId, const Ydb::ResultSet& resultSet, const TInstant& deadline, const TDuration& ttl) { NYql::TIssues issues; diff --git a/ydb/core/yq/libs/control_plane_storage/request_actor.h b/ydb/core/yq/libs/control_plane_storage/request_actor.h index 83f00e4fd89..7b40bbd00ec 100644 --- a/ydb/core/yq/libs/control_plane_storage/request_actor.h +++ b/ydb/core/yq/libs/control_plane_storage/request_actor.h @@ -7,7 +7,6 @@ #include <ydb/core/yq/libs/db_schema/db_schema.h> #include <ydb/core/yq/libs/shared_resources/db_exec.h> #include <ydb/public/api/protos/yq.pb.h> -#include <ydb/public/lib/yq/scope.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> #include <util/datetime/base.h> diff --git a/ydb/core/yq/libs/events/CMakeLists.darwin.txt b/ydb/core/yq/libs/events/CMakeLists.darwin.txt index ee4496c22e8..ddfd6a1e14f 100644 --- a/ydb/core/yq/libs/events/CMakeLists.darwin.txt +++ b/ydb/core/yq/libs/events/CMakeLists.darwin.txt @@ -19,12 +19,11 @@ target_link_libraries(yq-libs-events PUBLIC libs-graph_params-proto yq-libs-protos yql-core-facade + providers-common-db_id_async_resolver + providers-dq-provider yql-public-issue api-protos - public-lib-yq cpp-client-ydb_table - providers-common-db_id_async_resolver - providers-dq-provider ) generate_enum_serilization(yq-libs-events ${CMAKE_SOURCE_DIR}/ydb/core/yq/libs/events/events.h diff --git a/ydb/core/yq/libs/events/CMakeLists.linux-aarch64.txt b/ydb/core/yq/libs/events/CMakeLists.linux-aarch64.txt index 762d4b58fad..e238457ea44 100644 --- a/ydb/core/yq/libs/events/CMakeLists.linux-aarch64.txt +++ b/ydb/core/yq/libs/events/CMakeLists.linux-aarch64.txt @@ -20,12 +20,11 @@ target_link_libraries(yq-libs-events PUBLIC libs-graph_params-proto yq-libs-protos yql-core-facade + providers-common-db_id_async_resolver + providers-dq-provider yql-public-issue api-protos - public-lib-yq cpp-client-ydb_table - providers-common-db_id_async_resolver - providers-dq-provider ) generate_enum_serilization(yq-libs-events ${CMAKE_SOURCE_DIR}/ydb/core/yq/libs/events/events.h diff --git a/ydb/core/yq/libs/events/CMakeLists.linux.txt b/ydb/core/yq/libs/events/CMakeLists.linux.txt index 762d4b58fad..e238457ea44 100644 --- a/ydb/core/yq/libs/events/CMakeLists.linux.txt +++ b/ydb/core/yq/libs/events/CMakeLists.linux.txt @@ -20,12 +20,11 @@ target_link_libraries(yq-libs-events PUBLIC libs-graph_params-proto yq-libs-protos yql-core-facade + providers-common-db_id_async_resolver + providers-dq-provider yql-public-issue api-protos - public-lib-yq cpp-client-ydb_table - providers-common-db_id_async_resolver - providers-dq-provider ) generate_enum_serilization(yq-libs-events ${CMAKE_SOURCE_DIR}/ydb/core/yq/libs/events/events.h diff --git a/ydb/core/yq/libs/events/events.h b/ydb/core/yq/libs/events/events.h index e08b8efd45a..10ace97f109 100644 --- a/ydb/core/yq/libs/events/events.h +++ b/ydb/core/yq/libs/events/events.h @@ -9,7 +9,6 @@ #include <ydb/core/yq/libs/graph_params/proto/graph_params.pb.h> #include <ydb/core/yq/libs/protos/fq_private.pb.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> -#include <ydb/public/lib/yq/scope.h> #include <library/cpp/actors/core/events.h> @@ -17,8 +16,6 @@ namespace NYq { -using NYdb::NYq::TScope; - struct TQueryResult { TVector<Ydb::ResultSet> Sets; TInstant ExpirationDeadline; diff --git a/ydb/public/lib/fq/scope.cpp b/ydb/public/lib/fq/scope.cpp index fac8789e5f5..8bf4d3fd475 100644 --- a/ydb/public/lib/fq/scope.cpp +++ b/ydb/public/lib/fq/scope.cpp @@ -1,9 +1,19 @@ #include "scope.h" +#include <util/string/split.h> + namespace NYdb { namespace NFq { TString TScope::YandexCloudScopeSchema = "yandexcloud"; +TString TScope::ParseFolder() const { + const TVector<TString> path = StringSplitter(Scope).Split('/').SkipEmpty(); // yandexcloud://{folder_id} + if (path.size() == 2 && path.front().StartsWith(YandexCloudScopeSchema)) { + return path.back(); + } + return {}; +} + } // namespace NFq } // namespace Ndb diff --git a/ydb/public/lib/fq/scope.h b/ydb/public/lib/fq/scope.h index 4d54ea0a8e4..fe8eeebe5fb 100644 --- a/ydb/public/lib/fq/scope.h +++ b/ydb/public/lib/fq/scope.h @@ -28,6 +28,8 @@ public: return Scope.empty(); } + TString ParseFolder() const; + private: TString Scope; }; |