diff options
author | hcpp <hcpp@ydb.tech> | 2023-07-27 12:28:40 +0300 |
---|---|---|
committer | hcpp <hcpp@ydb.tech> | 2023-07-27 12:28:40 +0300 |
commit | dcde92436ae71c3fdf1d6b9916e3858f3b35146e (patch) | |
tree | c532626c2fcd831858aee80830c2514ebbe2a993 | |
parent | 565fe9166e5f84fa615bf4cd7b3bd4e3a9235f23 (diff) | |
download | ydb-dcde92436ae71c3fdf1d6b9916e3858f3b35146e.tar.gz |
external data source refactoring
25 files changed, 470 insertions, 241 deletions
diff --git a/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt index a3e2d9ccca..7893c75ba0 100644 --- a/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt @@ -24,6 +24,7 @@ target_link_libraries(core-kqp-gateway PUBLIC core-kqp-query_data kqp-gateway-actors gateway-behaviour-tablestore + gateway-behaviour-external_data_source providers-result-expr_nodes ) target_sources(core-kqp-gateway PRIVATE diff --git a/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt index bbb3eb0ee8..de6fc0d227 100644 --- a/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt @@ -25,6 +25,7 @@ target_link_libraries(core-kqp-gateway PUBLIC core-kqp-query_data kqp-gateway-actors gateway-behaviour-tablestore + gateway-behaviour-external_data_source providers-result-expr_nodes ) target_sources(core-kqp-gateway PRIVATE diff --git a/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt index bbb3eb0ee8..de6fc0d227 100644 --- a/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt +++ b/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt @@ -25,6 +25,7 @@ target_link_libraries(core-kqp-gateway PUBLIC core-kqp-query_data kqp-gateway-actors gateway-behaviour-tablestore + gateway-behaviour-external_data_source providers-result-expr_nodes ) target_sources(core-kqp-gateway PRIVATE diff --git a/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt index a3e2d9ccca..7893c75ba0 100644 --- a/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt +++ b/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt @@ -24,6 +24,7 @@ target_link_libraries(core-kqp-gateway PUBLIC core-kqp-query_data kqp-gateway-actors gateway-behaviour-tablestore + gateway-behaviour-external_data_source providers-result-expr_nodes ) target_sources(core-kqp-gateway PRIVATE diff --git a/ydb/core/kqp/gateway/behaviour/CMakeLists.txt b/ydb/core/kqp/gateway/behaviour/CMakeLists.txt index 096e41b032..2c9386bf8a 100644 --- a/ydb/core/kqp/gateway/behaviour/CMakeLists.txt +++ b/ydb/core/kqp/gateway/behaviour/CMakeLists.txt @@ -6,4 +6,5 @@ # original buildsystem will not be accepted. +add_subdirectory(external_data_source) add_subdirectory(tablestore) diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..aae8b0714b --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source PUBLIC + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +) + +add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source.global PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp +) diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..a8d7f24f00 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt @@ -0,0 +1,42 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +) + +add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source.global PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp +) diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..a8d7f24f00 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt @@ -0,0 +1,42 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +) + +add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source.global PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp +) diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..aae8b0714b --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source PUBLIC + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +) + +add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source) +target_compile_options(gateway-behaviour-external_data_source.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + kqp-gateway-actors + behaviour-tablestore-operations +) +target_sources(gateway-behaviour-external_data_source.global PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp +) diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp new file mode 100644 index 0000000000..33fc6519ee --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp @@ -0,0 +1,25 @@ +#include "behaviour.h" +#include "manager.h" + +namespace NKikimr::NKqp { + +TExternalDataSourceBehaviour::TFactory::TRegistrator<TExternalDataSourceBehaviour> TExternalDataSourceBehaviour::Registrator(TExternalDataSourceConfig::GetTypeId()); + +NMetadata::NModifications::IOperationsManager::TPtr TExternalDataSourceBehaviour::ConstructOperationsManager() const { + return std::make_shared<TExternalDataSourceManager>(); +} + +NMetadata::NInitializer::IInitializationBehaviour::TPtr TExternalDataSourceBehaviour::ConstructInitializer() const { + return nullptr; +} + +TString TExternalDataSourceBehaviour::GetInternalStorageTablePath() const { + return TExternalDataSourceConfig::GetTypeId(); +} + + +TString TExternalDataSourceBehaviour::GetTypeId() const { + return TExternalDataSourceConfig::GetTypeId(); +} + +} diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h new file mode 100644 index 0000000000..83ad16d89b --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h @@ -0,0 +1,25 @@ +#pragma once +#include <ydb/services/metadata/abstract/kqp_common.h> +#include <ydb/services/metadata/abstract/initialization.h> + +namespace NKikimr::NKqp { + +class TExternalDataSourceConfig { +public: + static TString GetTypeId() { + return "EXTERNAL_DATA_SOURCE"; + } +}; + +class TExternalDataSourceBehaviour: public NMetadata::TClassBehaviour<TExternalDataSourceConfig> { +private: + static TFactory::TRegistrator<TExternalDataSourceBehaviour> Registrator; +protected: + virtual std::shared_ptr<NMetadata::NInitializer::IInitializationBehaviour> ConstructInitializer() const override; + virtual std::shared_ptr<NMetadata::NModifications::IOperationsManager> ConstructOperationsManager() const override; + + virtual TString GetInternalStorageTablePath() const override; + virtual TString GetTypeId() const override; +}; + +} diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp new file mode 100644 index 0000000000..2dfc169914 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp @@ -0,0 +1,152 @@ + +#include "manager.h" + +#include <ydb/core/tx/tx_proxy/proxy.h> +#include <ydb/core/kqp/gateway/actors/scheme.h> +#include <ydb/core/kqp/provider/yql_kikimr_gateway.h> +#include <ydb/core/base/path.h> +#include <ydb/core/kqp/provider/yql_kikimr_gateway.h> + +#include <util/string/type.h> + +namespace NKikimr::NKqp { + +namespace { + +TString GetOrDefault(const NYql::TCreateObjectSettings& container, const TString& key, const TString& defaultValue = TString{}) { + auto fValue = container.GetFeaturesExtractor().Extract(key); + return fValue ? *fValue : defaultValue; +} + +void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescription& externaDataSourceDesc, + const TString& name, + const NYql::TCreateObjectSettings& settings) { + externaDataSourceDesc.SetName(name); + externaDataSourceDesc.SetSourceType(GetOrDefault(settings, "source_type")); + externaDataSourceDesc.SetLocation(GetOrDefault(settings, "location")); + externaDataSourceDesc.SetInstallation(GetOrDefault(settings, "installation")); + + TString authMethod = GetOrDefault(settings, "auth_method"); + if (authMethod == "NONE") { + externaDataSourceDesc.MutableAuth()->MutableNone(); + } else if (authMethod == "SERVICE_ACCOUNT") { + auto& sa = *externaDataSourceDesc.MutableAuth()->MutableServiceAccount(); + sa.SetId(GetOrDefault(settings, "service_account_id")); + sa.SetSecretName(GetOrDefault(settings, "service_account_secret_name")); + } +} + +NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvProposeTransaction* request, TActorSystem* actorSystem, bool failedOnAlreadyExists = false) +{ + auto promiseScheme = NThreading::NewPromise<NKqp::TSchemeOpRequestHandler::TResult>(); + IActor* requestHandler = new TSchemeOpRequestHandler(request, promiseScheme, failedOnAlreadyExists); + actorSystem->Register(requestHandler); + return promiseScheme.GetFuture().Apply([](const NThreading::TFuture<NKqp::TSchemeOpRequestHandler::TResult>& f) { + if (f.HasValue() && !f.HasException() && f.GetValue().Success()) { + return TExternalDataSourceManager::TYqlConclusionStatus::Success(); + } else if (f.HasValue()) { + return TExternalDataSourceManager::TYqlConclusionStatus::Fail(f.GetValue().Status(), f.GetValue().Issues().ToString()); + } + return TExternalDataSourceManager::TYqlConclusionStatus::Fail("no value in result"); + }); +} + +} + +NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> TExternalDataSourceManager::DoModify(const NYql::TObjectSettingsImpl& settings, + const ui32 nodeId, + NMetadata::IClassBehaviour::TPtr manager, + TInternalModificationContext& context) const { + Y_UNUSED(nodeId, manager, settings); + switch (context.GetActivityType()) { + case EActivityType::Upsert: + case EActivityType::Undefined: + case EActivityType::Alter: + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("not implemented")); + case EActivityType::Create: + return CreateExternalDataSource(settings, context); + case EActivityType::Drop: + return DropExternalDataSource(settings, context); + } +} + +NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> TExternalDataSourceManager::CreateExternalDataSource(const NYql::TObjectSettingsImpl& settings, + TInternalModificationContext& context) const { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + auto* actorSystem = context.GetExternalData().GetActorSystem(); + if (!actorSystem) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("This place needs an actor system. Please contact internal support")); + } + + if (!AppData(actorSystem)->FeatureFlags.GetEnableExternalDataSources()) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("External data sources are disabled. Please contact your system administrator to enable it")); + } + + std::pair<TString, TString> pathPair; + { + TString error; + if (!TrySplitPathByDb(settings.GetObjectId(), context.GetExternalData().GetDatabase(), pathPair, error)) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(error)); + } + } + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(context.GetExternalData().GetDatabase()); + if (context.GetExternalData().GetUserToken()) { + ev->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken()); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(pathPair.first); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateExternalDataSource); + + NKikimrSchemeOp::TExternalDataSourceDescription& dataSourceDesc = *schemeTx.MutableCreateExternalDataSource(); + FillCreateExternalDataSourceDesc(dataSourceDesc, pathPair.second, settings); + return SendSchemeRequest(ev.Release(), actorSystem, true); + } catch (...) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(CurrentExceptionMessage())); + } +} + +NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> TExternalDataSourceManager::DropExternalDataSource(const NYql::TObjectSettingsImpl& settings, + TInternalModificationContext& context) const { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + auto* actorSystem = context.GetExternalData().GetActorSystem(); + if (!actorSystem) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("This place needs an actor system. Please contact internal support")); + } + + if (!AppData(actorSystem)->FeatureFlags.GetEnableExternalDataSources()) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("External data sources are disabled. Please contact your system administrator to enable it")); + } + + std::pair<TString, TString> pathPair; + { + TString error; + if (!NYql::IKikimrGateway::TrySplitTablePath(settings.GetObjectId(), pathPair, error)) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(error)); + } + } + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(context.GetExternalData().GetDatabase()); + if (context.GetExternalData().GetUserToken()) { + ev->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken()); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(pathPair.first); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropExternalDataSource); + + NKikimrSchemeOp::TDrop& drop = *schemeTx.MutableDrop(); + drop.SetName(pathPair.second); + return SendSchemeRequest(ev.Release(), actorSystem); + } + catch (...) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(CurrentExceptionMessage())); + } +} + +} diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h new file mode 100644 index 0000000000..fb1045e203 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h @@ -0,0 +1,24 @@ +#pragma once + +#include <ydb/services/metadata/manager/generic_manager.h> + +namespace NKikimr::NKqp { + +class TExternalDataSourceManager: public NMetadata::NModifications::IOperationsManager { + NThreading::TFuture<TYqlConclusionStatus> CreateExternalDataSource(const NYql::TObjectSettingsImpl& settings, + TInternalModificationContext& context) const; + + NThreading::TFuture<TYqlConclusionStatus> DropExternalDataSource(const NYql::TObjectSettingsImpl& settings, + TInternalModificationContext& context) const; + +protected: + NThreading::TFuture<TYqlConclusionStatus> DoModify(const NYql::TObjectSettingsImpl& settings, + const ui32 nodeId, + NMetadata::IClassBehaviour::TPtr manager, + TInternalModificationContext& context) const override; + +public: + using NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus; +}; + +} diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make b/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make new file mode 100644 index 0000000000..2b52c19cc9 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +SRCS( + manager.cpp + GLOBAL behaviour.cpp +) + +PEERDIR( + ydb/services/metadata/initializer + ydb/services/metadata/abstract + ydb/core/kqp/gateway/actors + ydb/core/kqp/gateway/behaviour/tablestore/operations +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp index da3bd266c6..207e56c4a9 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp @@ -18,6 +18,11 @@ NThreading::TFuture<TTableStoreManager::TYqlConclusionStatus> TTableStoreManager auto promise = NThreading::NewPromise<TYqlConclusionStatus>(); auto result = promise.GetFuture(); + auto* actorSystem = context.GetExternalData().GetActorSystem(); + if (!actorSystem) { + return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("This place needs an actor system. Please contact internal support")); + } + switch (context.GetActivityType()) { case EActivityType::Create: case EActivityType::Upsert: @@ -48,8 +53,8 @@ NThreading::TFuture<TTableStoreManager::TYqlConclusionStatus> TTableStoreManager auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); operation->SerializeScheme(schemeTx); - auto promiseScheme = NThreading::NewPromise<NKqp::TSchemeOpRequestHandler::TResult>(); - TActivationContext::AsActorContext().Register(new NKqp::TSchemeOpRequestHandler(ev.Release(), promiseScheme, false)); + auto promiseScheme = NThreading::NewPromise<NKqp::TSchemeOpRequestHandler::TResult>(); + actorSystem->Register(new NKqp::TSchemeOpRequestHandler(ev.Release(), promiseScheme, false)); return promiseScheme.GetFuture().Apply([](const NThreading::TFuture<NKqp::TSchemeOpRequestHandler::TResult>& f) { if (f.HasValue() && !f.HasException() && f.GetValue().Success()) { return TYqlConclusionStatus::Success(); diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index 2cd2f11652..b685bde61f 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -1297,84 +1297,6 @@ public: } } - TFuture<TGenericResult> CreateExternalDataSource(const TString& cluster, - const NYql::TCreateExternalDataSourceSettings& settings, - bool createDir) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - std::pair<TString, TString> pathPair; - { - TString error; - if (!GetPathPair(settings.ExternalDataSource, pathPair, error, createDir)) { - return MakeFuture(ResultFromError<TGenericResult>(error)); - } - } - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(Database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->GetSerializedToken()); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(pathPair.first); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateExternalDataSource); - - NKikimrSchemeOp::TExternalDataSourceDescription& dataSourceDesc = *schemeTx.MutableCreateExternalDataSource(); - FillCreateExternalDataSourceDesc(dataSourceDesc, pathPair.second, settings); - return SendSchemeRequest(ev.Release(), true); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - - TFuture<TGenericResult> AlterExternalDataSource(const TString& cluster, - const NYql::TAlterExternalDataSourceSettings& settings) override { - Y_UNUSED(cluster, settings); - return MakeErrorFuture<TGenericResult>(std::make_exception_ptr(yexception() << "The alter is not supported for the external data source")); - } - - TFuture<TGenericResult> DropExternalDataSource(const TString& cluster, - const NYql::TDropExternalDataSourceSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - std::pair<TString, TString> pathPair; - { - TString error; - if (!GetPathPair(settings.ExternalDataSource, pathPair, error, false)) { - return MakeFuture(ResultFromError<TGenericResult>(error)); - } - } - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(Database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->GetSerializedToken()); - } - - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(pathPair.first); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropExternalDataSource); - - NKikimrSchemeOp::TDrop& drop = *schemeTx.MutableDrop(); - drop.SetName(pathPair.second); - return SendSchemeRequest(ev.Release()); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - TFuture<TGenericResult> AlterUser(const TString& cluster, const NYql::TAlterUserSettings& settings) override { using TRequest = TEvTxUserProxy::TEvProposeTransaction; @@ -1508,6 +1430,7 @@ public: context.SetUserToken(*GetUserToken()); } context.SetDatabase(Owner.Database); + context.SetActorSystem(Owner.ActorSystem); return DoExecute(cBehaviour, settings, context).Apply([](const NThreading::TFuture<TYqlConclusionStatus>& f) { if (f.HasValue() && !f.HasException() && f.GetValue().Ok()) { TGenericResult result; @@ -1801,7 +1724,7 @@ public: *taskResult.MutableItemType() = resultType; auto& taskConnection = *taskResult.MutableConnection(); taskConnection.SetStageIndex(0); - + NKikimr::NKqp::TPreparedQueryHolder queryHolder(preparedQuery.release(), txAlloc->HolderFactory.GetFunctionRegistry()); NKikimr::NKqp::TQueryData::TPtr params = std::make_shared<NKikimr::NKqp::TQueryData>(txAlloc); @@ -2270,24 +2193,6 @@ private: externalTableDesc.SetContent(general.SerializeAsString()); } - static void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescription& externaDataSourceDesc, - const TString& name, - const NYql::TCreateExternalDataSourceSettings& settings) - { - externaDataSourceDesc.SetName(name); - externaDataSourceDesc.SetSourceType(settings.SourceType); - externaDataSourceDesc.SetLocation(settings.Location); - externaDataSourceDesc.SetInstallation(settings.Installation); - - if (settings.AuthMethod == "NONE") { - externaDataSourceDesc.MutableAuth()->MutableNone(); - } else if (settings.AuthMethod == "SERVICE_ACCOUNT") { - auto& sa = *externaDataSourceDesc.MutableAuth()->MutableServiceAccount(); - sa.SetId(settings.ServiceAccount.Id); - sa.SetSecretName(settings.ServiceAccount.SecretName); - } - } - static void FillParameters(TQueryData::TPtr params, ::google::protobuf::Map<TBasicString<char>, Ydb::TypedValue>* output) { if (!params) { return; diff --git a/ydb/core/kqp/gateway/ya.make b/ydb/core/kqp/gateway/ya.make index 684bc256ec..ad479b21a9 100644 --- a/ydb/core/kqp/gateway/ya.make +++ b/ydb/core/kqp/gateway/ya.make @@ -15,6 +15,7 @@ PEERDIR( ydb/core/kqp/query_data ydb/core/kqp/gateway/actors ydb/core/kqp/gateway/behaviour/tablestore + ydb/core/kqp/gateway/behaviour/external_data_source ydb/library/yql/providers/result/expr_nodes ) diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 65707bb83c..b17d6d8d97 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -640,35 +640,16 @@ public: FORWARD_ENSURE_NO_PREPARE(DropExternalTable, cluster, settings); } - TFuture<TGenericResult> CreateExternalDataSource(const TString& cluster, - const TCreateExternalDataSourceSettings& settings, bool createDir) override - { - FORWARD_ENSURE_NO_PREPARE(CreateExternalDataSource, cluster, settings, createDir); - } - - TFuture<TGenericResult> AlterExternalDataSource(const TString& cluster, - const TAlterExternalDataSourceSettings& settings) override - { - FORWARD_ENSURE_NO_PREPARE(AlterExternalDataSource, cluster, settings); - } - - TFuture<TGenericResult> DropExternalDataSource(const TString& cluster, - const TDropExternalDataSourceSettings& settings) override - { - FORWARD_ENSURE_NO_PREPARE(DropExternalDataSource, cluster, settings); - } - TVector<TString> GetCollectedSchemeData() override { return Gateway->GetCollectedSchemeData(); } - TFuture<TExecuteLiteralResult> ExecuteLiteral(const TString& program, - const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) override + TFuture<TExecuteLiteralResult> ExecuteLiteral(const TString& program, + const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) override { return Gateway->ExecuteLiteral(program, resultType, txAlloc); } - private: bool IsPrepare() const { if (!SessionCtx) { diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 3e8d8fd27d..ec0792a718 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -179,35 +179,6 @@ namespace { return dropGroupSettings; } - TString GetOrDefault(const TCreateObjectSettings& container, const TString& key, const TString& defaultValue = TString{}) { - auto fValue = container.GetFeaturesExtractor().Extract(key); - return fValue ? *fValue : defaultValue; - } - - TCreateExternalDataSourceSettings ParseCreateExternalDataSourceSettings(const TCreateObjectSettings& settings) { - TCreateExternalDataSourceSettings out; - out.ExternalDataSource = settings.GetObjectId(); - out.SourceType = GetOrDefault(settings, "source_type"); - out.AuthMethod = GetOrDefault(settings, "auth_method"); - out.ServiceAccount.Id = GetOrDefault(settings, "service_account_id"); - out.ServiceAccount.SecretName = GetOrDefault(settings, "service_account_secret_name"); - out.Installation = GetOrDefault(settings, "installation"); - out.Location = GetOrDefault(settings, "location"); - return out; - } - - TAlterExternalDataSourceSettings ParseAlterExternalDataSourceSettings(const TAlterObjectSettings& settings) { - TAlterExternalDataSourceSettings out; - out.ExternalDataSource = settings.GetObjectId(); - return out; - } - - TDropExternalDataSourceSettings ParseDropExternalDataSourceSettings(const TDropObjectSettings& settings) { - TDropExternalDataSourceSettings out; - out.ExternalDataSource = settings.GetObjectId(); - return out; - } - TCreateTableStoreSettings ParseCreateTableStoreSettings(TKiCreateTable create, const TTableSettings& settings) { TCreateTableStoreSettings out; out.TableStore = TString(create.Table()); @@ -564,7 +535,7 @@ private: *SessionCtx->Query().QueryData->GetAllocState()->HolderFactory.GetFunctionRegistry(), ctx, fakeReads); - NKikimr::NMiniKQL::TProgramBuilder programBuilder(SessionCtx->Query().QueryData->GetAllocState()->TypeEnv, + NKikimr::NMiniKQL::TProgramBuilder programBuilder(SessionCtx->Query().QueryData->GetAllocState()->TypeEnv, *SessionCtx->Query().QueryData->GetAllocState()->HolderFactory.GetFunctionRegistry()); TStringStream errorStream; @@ -584,7 +555,7 @@ private: NYson::TYsonWriter writer2((IOutputStream*)&out); writer2.OnBeginMap(); writer2.OnKeyedItem("Data"); - writer2.OnRaw(ysonStream.Str()); + writer2.OnRaw(ysonStream.Str()); writer2.OnEndMap(); return out.Str(); @@ -867,48 +838,6 @@ public: using TBase::TBase; }; -class TCreateExternalDataSourceTransformer: public TObjectModifierTransformer<TKiCreateObject, TCreateObjectSettings> { -private: - using TBase = TObjectModifierTransformer<TKiCreateObject, TCreateObjectSettings>; -protected: - virtual TFuture<IKikimrGateway::TGenericResult> DoExecute(const TString& cluster, const TCreateObjectSettings& settings) override { - if (!SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) { - return MakeErrorFuture<IKikimrGateway::TGenericResult>(std::make_exception_ptr(yexception() << "External data sources are disabled. Please contact your system administrator to enable it")); - } - return GetGateway()->CreateExternalDataSource(cluster, ParseCreateExternalDataSourceSettings(settings), true); - } -public: - using TBase::TBase; -}; - -class TAlterExternalDataSourceTransformer: public TObjectModifierTransformer<TKiAlterObject, TAlterObjectSettings> { -private: - using TBase = TObjectModifierTransformer<TKiAlterObject, TAlterObjectSettings>; -protected: - virtual TFuture<IKikimrGateway::TGenericResult> DoExecute(const TString& cluster, const TAlterObjectSettings& settings) override { - if (!SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) { - return MakeErrorFuture<IKikimrGateway::TGenericResult>(std::make_exception_ptr(yexception() << "External data sources are disabled. Please contact your system administrator to enable it")); - } - return GetGateway()->AlterExternalDataSource(cluster, ParseAlterExternalDataSourceSettings(settings)); - } -public: - using TBase::TBase; -}; - -class TDropExternalDataSourceTransformer: public TObjectModifierTransformer<TKiDropObject, TDropObjectSettings> { -private: - using TBase = TObjectModifierTransformer<TKiDropObject, TDropObjectSettings>; -protected: - virtual TFuture<IKikimrGateway::TGenericResult> DoExecute(const TString& cluster, const TDropObjectSettings& settings) override { - if (!SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) { - return MakeErrorFuture<IKikimrGateway::TGenericResult>(std::make_exception_ptr(yexception() << "External data sources are disabled. Please contact your system administrator to enable it")); - } - return GetGateway()->DropExternalDataSource(cluster, ParseDropExternalDataSourceSettings(settings)); - } -public: - using TBase::TBase; -}; - class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKiSinkCallableExecutionTransformer> { public: TKiSinkCallableExecutionTransformer( @@ -1774,21 +1703,15 @@ public: } if (auto kiObject = TMaybeNode<TKiCreateObject>(input)) { - return kiObject.Cast().TypeId() == "EXTERNAL_DATA_SOURCE" - ? TCreateExternalDataSourceTransformer("CREATE EXTERNAL DATA SOURCE", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx) - : TCreateObjectTransformer("CREATE OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx); + return TCreateObjectTransformer("CREATE OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx); } if (auto kiObject = TMaybeNode<TKiAlterObject>(input)) { - return kiObject.Cast().TypeId() == "EXTERNAL_DATA_SOURCE" - ? TAlterExternalDataSourceTransformer("ALTER EXTERNAL DATA SOURCE", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx) - : TAlterObjectTransformer("ALTER OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx); + return TAlterObjectTransformer("ALTER OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx); } if (auto kiObject = TMaybeNode<TKiDropObject>(input)) { - return kiObject.Cast().TypeId() == "EXTERNAL_DATA_SOURCE" - ? TDropExternalDataSourceTransformer("DROP EXTERNAL DATA SOURCE", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx) - : TDropObjectTransformer("DROP OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx); + return TDropObjectTransformer("DROP OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx); } if (auto maybeCreateGroup = TMaybeNode<TKiCreateGroup>(input)) { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 60bbcff13e..f5076573d1 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -587,28 +587,6 @@ struct TDropExternalTableSettings { TString ExternalTable; }; -struct TCreateExternalDataSourceSettings { - struct TServiceAccount { - TString Id; - TString SecretName; - }; - - TString ExternalDataSource; - TString SourceType; - TString Location; - TString Installation; - TString AuthMethod; - TServiceAccount ServiceAccount; -}; - -struct TAlterExternalDataSourceSettings { - TString ExternalDataSource; -}; - -struct TDropExternalDataSourceSettings { - TString ExternalDataSource; -}; - struct TKikimrListPathItem { TKikimrListPathItem(TString name, bool isDirectory) { Name = name; @@ -791,12 +769,6 @@ public: virtual NThreading::TFuture<TGenericResult> DropExternalTable(const TString& cluster, const TDropExternalTableSettings& settings) = 0; - virtual NThreading::TFuture<TGenericResult> CreateExternalDataSource(const TString& cluster, const TCreateExternalDataSourceSettings& settings, bool createDir) = 0; - - virtual NThreading::TFuture<TGenericResult> AlterExternalDataSource(const TString& cluster, const TAlterExternalDataSourceSettings& settings) = 0; - - virtual NThreading::TFuture<TGenericResult> DropExternalDataSource(const TString& cluster, const TDropExternalDataSourceSettings& settings) = 0; - virtual TVector<TString> GetCollectedSchemeData() = 0; virtual NThreading::TFuture<TExecuteLiteralResult> ExecuteLiteral(const TString& program, const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) = 0; diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index a390e09ac6..a96701e163 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -134,13 +134,12 @@ void TestDropTableCommon(TIntrusivePtr<IKikimrGateway> gateway) { } void TestCreateExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) { - NYql::TCreateExternalDataSourceSettings settings; - settings.ExternalDataSource = path; - settings.SourceType = "ObjectStorage"; - settings.AuthMethod = "NONE"; - settings.Installation = "cloud"; - - auto responseFuture = gateway->CreateExternalDataSource(TestCluster, settings, true); + TCreateObjectSettings settings("EXTERNAL_DATA_SOURCE", path, { + {"source_type", "ObjectStorage"}, + {"auth_method", "NONE"}, + {"installation", "cloud"} + }); + auto responseFuture = gateway->CreateObject(TestCluster, settings); responseFuture.Wait(); auto response = responseFuture.GetValue(); response.Issues().PrintTo(Cerr); @@ -160,7 +159,6 @@ void TestCreateExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKik void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path, bool fail = false) { NYql::TCreateExternalTableSettings settings; - settings.ExternalTable = path; settings.DataSourcePath = "/Root/f1/f2/external_data_source"; settings.Location = "/"; @@ -191,7 +189,6 @@ void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGa } void TestDropExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) { - auto responseFuture = gateway->DropExternalTable(TestCluster, TDropExternalTableSettings{.ExternalTable=path}); responseFuture.Wait(); auto response = responseFuture.GetValue(); @@ -205,7 +202,8 @@ void TestDropExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGate } void TestDropExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) { - auto responseFuture = gateway->DropExternalDataSource(TestCluster, TDropExternalDataSourceSettings{.ExternalDataSource=path}); + TDropObjectSettings settings("EXTERNAL_DATA_SOURCE", path, {}); + auto responseFuture = gateway->DropObject(TestCluster, settings); responseFuture.Wait(); auto response = responseFuture.GetValue(); response.Issues().PrintTo(Cerr); @@ -241,12 +239,14 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { Y_UNIT_TEST(TestCreateExternalTable) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source"); TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table"); } Y_UNIT_TEST(TestCreateSameExternalTable) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source"); TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table"); TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table", true); @@ -254,6 +254,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { Y_UNIT_TEST(TestDropExternalTable) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source"); TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table"); TestDropExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table"); @@ -261,6 +262,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { Y_UNIT_TEST(TestDropExternalDataSource) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source"); TestDropExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source"); } diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 5578d6a69f..05974c28bb 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -4476,7 +4476,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { AUTH_METHOD="NONE" );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::INTERNAL_ERROR); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "External data sources are disabled. Please contact your system administrator to enable it"); } diff --git a/ydb/services/metadata/abstract/parsing.h b/ydb/services/metadata/abstract/parsing.h index 44db7b66f1..433bd6828b 100644 --- a/ydb/services/metadata/abstract/parsing.h +++ b/ydb/services/metadata/abstract/parsing.h @@ -18,6 +18,14 @@ private: TFeatures Features; std::shared_ptr<TFeaturesExtractor> FeaturesExtractor; public: + TObjectSettingsImpl() = default; + + TObjectSettingsImpl(const TString& typeId, const TString& objectId, const TFeatures& features) + : TypeId(typeId) + , ObjectId(objectId) + , Features(features) + , FeaturesExtractor(std::make_shared<TFeaturesExtractor>(Features)) + {} TFeaturesExtractor& GetFeaturesExtractor() const { Y_VERIFY(!!FeaturesExtractor); diff --git a/ydb/services/metadata/manager/abstract.h b/ydb/services/metadata/manager/abstract.h index 48b884f53c..b03cc8bbb2 100644 --- a/ydb/services/metadata/manager/abstract.h +++ b/ydb/services/metadata/manager/abstract.h @@ -14,6 +14,7 @@ #include <ydb/services/metadata/abstract/parsing.h> #include <library/cpp/threading/future/core/future.h> +#include <library/cpp/actors/core/actorsystem.h> namespace NKikimr::NMetadata::NModifications { @@ -62,6 +63,8 @@ public: private: YDB_ACCESSOR_DEF(std::optional<NACLib::TUserToken>, UserToken); YDB_ACCESSOR_DEF(TString, Database); + using TActorSystemPtr = TActorSystem*; + YDB_ACCESSOR_DEF(TActorSystemPtr, ActorSystem); }; class TInternalModificationContext { |