aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhcpp <hcpp@ydb.tech>2023-07-27 12:28:40 +0300
committerhcpp <hcpp@ydb.tech>2023-07-27 12:28:40 +0300
commitdcde92436ae71c3fdf1d6b9916e3858f3b35146e (patch)
treec532626c2fcd831858aee80830c2514ebbe2a993
parent565fe9166e5f84fa615bf4cd7b3bd4e3a9235f23 (diff)
downloadydb-dcde92436ae71c3fdf1d6b9916e3858f3b35146e.tar.gz
external data source refactoring
-rw-r--r--ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/core/kqp/gateway/behaviour/CMakeLists.txt1
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt40
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt42
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt42
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt17
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt40
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp25
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h25
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp152
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/manager.h24
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/ya.make17
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp9
-rw-r--r--ydb/core/kqp/gateway/kqp_ic_gateway.cpp99
-rw-r--r--ydb/core/kqp/gateway/ya.make1
-rw-r--r--ydb/core/kqp/host/kqp_gateway_proxy.cpp23
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_exec.cpp87
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway.h28
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp22
-rw-r--r--ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp2
-rw-r--r--ydb/services/metadata/abstract/parsing.h8
-rw-r--r--ydb/services/metadata/manager/abstract.h3
25 files changed, 470 insertions, 241 deletions
diff --git a/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt
index a3e2d9ccca..7893c75ba0 100644
--- a/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/kqp/gateway/CMakeLists.darwin-x86_64.txt
@@ -24,6 +24,7 @@ target_link_libraries(core-kqp-gateway PUBLIC
core-kqp-query_data
kqp-gateway-actors
gateway-behaviour-tablestore
+ gateway-behaviour-external_data_source
providers-result-expr_nodes
)
target_sources(core-kqp-gateway PRIVATE
diff --git a/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt
index bbb3eb0ee8..de6fc0d227 100644
--- a/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/kqp/gateway/CMakeLists.linux-aarch64.txt
@@ -25,6 +25,7 @@ target_link_libraries(core-kqp-gateway PUBLIC
core-kqp-query_data
kqp-gateway-actors
gateway-behaviour-tablestore
+ gateway-behaviour-external_data_source
providers-result-expr_nodes
)
target_sources(core-kqp-gateway PRIVATE
diff --git a/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt
index bbb3eb0ee8..de6fc0d227 100644
--- a/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/kqp/gateway/CMakeLists.linux-x86_64.txt
@@ -25,6 +25,7 @@ target_link_libraries(core-kqp-gateway PUBLIC
core-kqp-query_data
kqp-gateway-actors
gateway-behaviour-tablestore
+ gateway-behaviour-external_data_source
providers-result-expr_nodes
)
target_sources(core-kqp-gateway PRIVATE
diff --git a/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt
index a3e2d9ccca..7893c75ba0 100644
--- a/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/kqp/gateway/CMakeLists.windows-x86_64.txt
@@ -24,6 +24,7 @@ target_link_libraries(core-kqp-gateway PUBLIC
core-kqp-query_data
kqp-gateway-actors
gateway-behaviour-tablestore
+ gateway-behaviour-external_data_source
providers-result-expr_nodes
)
target_sources(core-kqp-gateway PRIVATE
diff --git a/ydb/core/kqp/gateway/behaviour/CMakeLists.txt b/ydb/core/kqp/gateway/behaviour/CMakeLists.txt
index 096e41b032..2c9386bf8a 100644
--- a/ydb/core/kqp/gateway/behaviour/CMakeLists.txt
+++ b/ydb/core/kqp/gateway/behaviour/CMakeLists.txt
@@ -6,4 +6,5 @@
# original buildsystem will not be accepted.
+add_subdirectory(external_data_source)
add_subdirectory(tablestore)
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..aae8b0714b
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
+)
+
+add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp
+)
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..a8d7f24f00
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,42 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
+)
+
+add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp
+)
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..a8d7f24f00
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,42 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
+)
+
+add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp
+)
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..aae8b0714b
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
+)
+
+add_global_library_for(gateway-behaviour-external_data_source.global gateway-behaviour-external_data_source)
+target_compile_options(gateway-behaviour-external_data_source.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(gateway-behaviour-external_data_source.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ services-metadata-initializer
+ services-metadata-abstract
+ kqp-gateway-actors
+ behaviour-tablestore-operations
+)
+target_sources(gateway-behaviour-external_data_source.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp
+)
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp
new file mode 100644
index 0000000000..33fc6519ee
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.cpp
@@ -0,0 +1,25 @@
+#include "behaviour.h"
+#include "manager.h"
+
+namespace NKikimr::NKqp {
+
+TExternalDataSourceBehaviour::TFactory::TRegistrator<TExternalDataSourceBehaviour> TExternalDataSourceBehaviour::Registrator(TExternalDataSourceConfig::GetTypeId());
+
+NMetadata::NModifications::IOperationsManager::TPtr TExternalDataSourceBehaviour::ConstructOperationsManager() const {
+ return std::make_shared<TExternalDataSourceManager>();
+}
+
+NMetadata::NInitializer::IInitializationBehaviour::TPtr TExternalDataSourceBehaviour::ConstructInitializer() const {
+ return nullptr;
+}
+
+TString TExternalDataSourceBehaviour::GetInternalStorageTablePath() const {
+ return TExternalDataSourceConfig::GetTypeId();
+}
+
+
+TString TExternalDataSourceBehaviour::GetTypeId() const {
+ return TExternalDataSourceConfig::GetTypeId();
+}
+
+}
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h
new file mode 100644
index 0000000000..83ad16d89b
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/behaviour.h
@@ -0,0 +1,25 @@
+#pragma once
+#include <ydb/services/metadata/abstract/kqp_common.h>
+#include <ydb/services/metadata/abstract/initialization.h>
+
+namespace NKikimr::NKqp {
+
+class TExternalDataSourceConfig {
+public:
+ static TString GetTypeId() {
+ return "EXTERNAL_DATA_SOURCE";
+ }
+};
+
+class TExternalDataSourceBehaviour: public NMetadata::TClassBehaviour<TExternalDataSourceConfig> {
+private:
+ static TFactory::TRegistrator<TExternalDataSourceBehaviour> Registrator;
+protected:
+ virtual std::shared_ptr<NMetadata::NInitializer::IInitializationBehaviour> ConstructInitializer() const override;
+ virtual std::shared_ptr<NMetadata::NModifications::IOperationsManager> ConstructOperationsManager() const override;
+
+ virtual TString GetInternalStorageTablePath() const override;
+ virtual TString GetTypeId() const override;
+};
+
+}
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
new file mode 100644
index 0000000000..2dfc169914
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
@@ -0,0 +1,152 @@
+
+#include "manager.h"
+
+#include <ydb/core/tx/tx_proxy/proxy.h>
+#include <ydb/core/kqp/gateway/actors/scheme.h>
+#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
+#include <ydb/core/base/path.h>
+#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
+
+#include <util/string/type.h>
+
+namespace NKikimr::NKqp {
+
+namespace {
+
+TString GetOrDefault(const NYql::TCreateObjectSettings& container, const TString& key, const TString& defaultValue = TString{}) {
+ auto fValue = container.GetFeaturesExtractor().Extract(key);
+ return fValue ? *fValue : defaultValue;
+}
+
+void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescription& externaDataSourceDesc,
+ const TString& name,
+ const NYql::TCreateObjectSettings& settings) {
+ externaDataSourceDesc.SetName(name);
+ externaDataSourceDesc.SetSourceType(GetOrDefault(settings, "source_type"));
+ externaDataSourceDesc.SetLocation(GetOrDefault(settings, "location"));
+ externaDataSourceDesc.SetInstallation(GetOrDefault(settings, "installation"));
+
+ TString authMethod = GetOrDefault(settings, "auth_method");
+ if (authMethod == "NONE") {
+ externaDataSourceDesc.MutableAuth()->MutableNone();
+ } else if (authMethod == "SERVICE_ACCOUNT") {
+ auto& sa = *externaDataSourceDesc.MutableAuth()->MutableServiceAccount();
+ sa.SetId(GetOrDefault(settings, "service_account_id"));
+ sa.SetSecretName(GetOrDefault(settings, "service_account_secret_name"));
+ }
+}
+
+NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvProposeTransaction* request, TActorSystem* actorSystem, bool failedOnAlreadyExists = false)
+{
+ auto promiseScheme = NThreading::NewPromise<NKqp::TSchemeOpRequestHandler::TResult>();
+ IActor* requestHandler = new TSchemeOpRequestHandler(request, promiseScheme, failedOnAlreadyExists);
+ actorSystem->Register(requestHandler);
+ return promiseScheme.GetFuture().Apply([](const NThreading::TFuture<NKqp::TSchemeOpRequestHandler::TResult>& f) {
+ if (f.HasValue() && !f.HasException() && f.GetValue().Success()) {
+ return TExternalDataSourceManager::TYqlConclusionStatus::Success();
+ } else if (f.HasValue()) {
+ return TExternalDataSourceManager::TYqlConclusionStatus::Fail(f.GetValue().Status(), f.GetValue().Issues().ToString());
+ }
+ return TExternalDataSourceManager::TYqlConclusionStatus::Fail("no value in result");
+ });
+}
+
+}
+
+NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> TExternalDataSourceManager::DoModify(const NYql::TObjectSettingsImpl& settings,
+ const ui32 nodeId,
+ NMetadata::IClassBehaviour::TPtr manager,
+ TInternalModificationContext& context) const {
+ Y_UNUSED(nodeId, manager, settings);
+ switch (context.GetActivityType()) {
+ case EActivityType::Upsert:
+ case EActivityType::Undefined:
+ case EActivityType::Alter:
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("not implemented"));
+ case EActivityType::Create:
+ return CreateExternalDataSource(settings, context);
+ case EActivityType::Drop:
+ return DropExternalDataSource(settings, context);
+ }
+}
+
+NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> TExternalDataSourceManager::CreateExternalDataSource(const NYql::TObjectSettingsImpl& settings,
+ TInternalModificationContext& context) const {
+ using TRequest = TEvTxUserProxy::TEvProposeTransaction;
+
+ try {
+ auto* actorSystem = context.GetExternalData().GetActorSystem();
+ if (!actorSystem) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("This place needs an actor system. Please contact internal support"));
+ }
+
+ if (!AppData(actorSystem)->FeatureFlags.GetEnableExternalDataSources()) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("External data sources are disabled. Please contact your system administrator to enable it"));
+ }
+
+ std::pair<TString, TString> pathPair;
+ {
+ TString error;
+ if (!TrySplitPathByDb(settings.GetObjectId(), context.GetExternalData().GetDatabase(), pathPair, error)) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(error));
+ }
+ }
+
+ auto ev = MakeHolder<TRequest>();
+ ev->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
+ if (context.GetExternalData().GetUserToken()) {
+ ev->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken());
+ }
+ auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme();
+ schemeTx.SetWorkingDir(pathPair.first);
+ schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateExternalDataSource);
+
+ NKikimrSchemeOp::TExternalDataSourceDescription& dataSourceDesc = *schemeTx.MutableCreateExternalDataSource();
+ FillCreateExternalDataSourceDesc(dataSourceDesc, pathPair.second, settings);
+ return SendSchemeRequest(ev.Release(), actorSystem, true);
+ } catch (...) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(CurrentExceptionMessage()));
+ }
+}
+
+NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> TExternalDataSourceManager::DropExternalDataSource(const NYql::TObjectSettingsImpl& settings,
+ TInternalModificationContext& context) const {
+ using TRequest = TEvTxUserProxy::TEvProposeTransaction;
+
+ try {
+ auto* actorSystem = context.GetExternalData().GetActorSystem();
+ if (!actorSystem) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("This place needs an actor system. Please contact internal support"));
+ }
+
+ if (!AppData(actorSystem)->FeatureFlags.GetEnableExternalDataSources()) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("External data sources are disabled. Please contact your system administrator to enable it"));
+ }
+
+ std::pair<TString, TString> pathPair;
+ {
+ TString error;
+ if (!NYql::IKikimrGateway::TrySplitTablePath(settings.GetObjectId(), pathPair, error)) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(error));
+ }
+ }
+
+ auto ev = MakeHolder<TRequest>();
+ ev->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
+ if (context.GetExternalData().GetUserToken()) {
+ ev->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken());
+ }
+ auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme();
+ schemeTx.SetWorkingDir(pathPair.first);
+ schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropExternalDataSource);
+
+ NKikimrSchemeOp::TDrop& drop = *schemeTx.MutableDrop();
+ drop.SetName(pathPair.second);
+ return SendSchemeRequest(ev.Release(), actorSystem);
+ }
+ catch (...) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail(CurrentExceptionMessage()));
+ }
+}
+
+}
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h
new file mode 100644
index 0000000000..fb1045e203
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <ydb/services/metadata/manager/generic_manager.h>
+
+namespace NKikimr::NKqp {
+
+class TExternalDataSourceManager: public NMetadata::NModifications::IOperationsManager {
+ NThreading::TFuture<TYqlConclusionStatus> CreateExternalDataSource(const NYql::TObjectSettingsImpl& settings,
+ TInternalModificationContext& context) const;
+
+ NThreading::TFuture<TYqlConclusionStatus> DropExternalDataSource(const NYql::TObjectSettingsImpl& settings,
+ TInternalModificationContext& context) const;
+
+protected:
+ NThreading::TFuture<TYqlConclusionStatus> DoModify(const NYql::TObjectSettingsImpl& settings,
+ const ui32 nodeId,
+ NMetadata::IClassBehaviour::TPtr manager,
+ TInternalModificationContext& context) const override;
+
+public:
+ using NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus;
+};
+
+}
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make b/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make
new file mode 100644
index 0000000000..2b52c19cc9
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+SRCS(
+ manager.cpp
+ GLOBAL behaviour.cpp
+)
+
+PEERDIR(
+ ydb/services/metadata/initializer
+ ydb/services/metadata/abstract
+ ydb/core/kqp/gateway/actors
+ ydb/core/kqp/gateway/behaviour/tablestore/operations
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp
index da3bd266c6..207e56c4a9 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp
@@ -18,6 +18,11 @@ NThreading::TFuture<TTableStoreManager::TYqlConclusionStatus> TTableStoreManager
auto promise = NThreading::NewPromise<TYqlConclusionStatus>();
auto result = promise.GetFuture();
+ auto* actorSystem = context.GetExternalData().GetActorSystem();
+ if (!actorSystem) {
+ return NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Fail("This place needs an actor system. Please contact internal support"));
+ }
+
switch (context.GetActivityType()) {
case EActivityType::Create:
case EActivityType::Upsert:
@@ -48,8 +53,8 @@ NThreading::TFuture<TTableStoreManager::TYqlConclusionStatus> TTableStoreManager
auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme();
operation->SerializeScheme(schemeTx);
- auto promiseScheme = NThreading::NewPromise<NKqp::TSchemeOpRequestHandler::TResult>();
- TActivationContext::AsActorContext().Register(new NKqp::TSchemeOpRequestHandler(ev.Release(), promiseScheme, false));
+ auto promiseScheme = NThreading::NewPromise<NKqp::TSchemeOpRequestHandler::TResult>();
+ actorSystem->Register(new NKqp::TSchemeOpRequestHandler(ev.Release(), promiseScheme, false));
return promiseScheme.GetFuture().Apply([](const NThreading::TFuture<NKqp::TSchemeOpRequestHandler::TResult>& f) {
if (f.HasValue() && !f.HasException() && f.GetValue().Success()) {
return TYqlConclusionStatus::Success();
diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
index 2cd2f11652..b685bde61f 100644
--- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
+++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
@@ -1297,84 +1297,6 @@ public:
}
}
- TFuture<TGenericResult> CreateExternalDataSource(const TString& cluster,
- const NYql::TCreateExternalDataSourceSettings& settings,
- bool createDir) override {
- using TRequest = TEvTxUserProxy::TEvProposeTransaction;
-
- try {
- if (!CheckCluster(cluster)) {
- return InvalidCluster<TGenericResult>(cluster);
- }
-
- std::pair<TString, TString> pathPair;
- {
- TString error;
- if (!GetPathPair(settings.ExternalDataSource, pathPair, error, createDir)) {
- return MakeFuture(ResultFromError<TGenericResult>(error));
- }
- }
-
- auto ev = MakeHolder<TRequest>();
- ev->Record.SetDatabaseName(Database);
- if (UserToken) {
- ev->Record.SetUserToken(UserToken->GetSerializedToken());
- }
- auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme();
- schemeTx.SetWorkingDir(pathPair.first);
- schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateExternalDataSource);
-
- NKikimrSchemeOp::TExternalDataSourceDescription& dataSourceDesc = *schemeTx.MutableCreateExternalDataSource();
- FillCreateExternalDataSourceDesc(dataSourceDesc, pathPair.second, settings);
- return SendSchemeRequest(ev.Release(), true);
- }
- catch (yexception& e) {
- return MakeFuture(ResultFromException<TGenericResult>(e));
- }
- }
-
- TFuture<TGenericResult> AlterExternalDataSource(const TString& cluster,
- const NYql::TAlterExternalDataSourceSettings& settings) override {
- Y_UNUSED(cluster, settings);
- return MakeErrorFuture<TGenericResult>(std::make_exception_ptr(yexception() << "The alter is not supported for the external data source"));
- }
-
- TFuture<TGenericResult> DropExternalDataSource(const TString& cluster,
- const NYql::TDropExternalDataSourceSettings& settings) override {
- using TRequest = TEvTxUserProxy::TEvProposeTransaction;
-
- try {
- if (!CheckCluster(cluster)) {
- return InvalidCluster<TGenericResult>(cluster);
- }
-
- std::pair<TString, TString> pathPair;
- {
- TString error;
- if (!GetPathPair(settings.ExternalDataSource, pathPair, error, false)) {
- return MakeFuture(ResultFromError<TGenericResult>(error));
- }
- }
-
- auto ev = MakeHolder<TRequest>();
- ev->Record.SetDatabaseName(Database);
- if (UserToken) {
- ev->Record.SetUserToken(UserToken->GetSerializedToken());
- }
-
- auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme();
- schemeTx.SetWorkingDir(pathPair.first);
- schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropExternalDataSource);
-
- NKikimrSchemeOp::TDrop& drop = *schemeTx.MutableDrop();
- drop.SetName(pathPair.second);
- return SendSchemeRequest(ev.Release());
- }
- catch (yexception& e) {
- return MakeFuture(ResultFromException<TGenericResult>(e));
- }
- }
-
TFuture<TGenericResult> AlterUser(const TString& cluster, const NYql::TAlterUserSettings& settings) override {
using TRequest = TEvTxUserProxy::TEvProposeTransaction;
@@ -1508,6 +1430,7 @@ public:
context.SetUserToken(*GetUserToken());
}
context.SetDatabase(Owner.Database);
+ context.SetActorSystem(Owner.ActorSystem);
return DoExecute(cBehaviour, settings, context).Apply([](const NThreading::TFuture<TYqlConclusionStatus>& f) {
if (f.HasValue() && !f.HasException() && f.GetValue().Ok()) {
TGenericResult result;
@@ -1801,7 +1724,7 @@ public:
*taskResult.MutableItemType() = resultType;
auto& taskConnection = *taskResult.MutableConnection();
taskConnection.SetStageIndex(0);
-
+
NKikimr::NKqp::TPreparedQueryHolder queryHolder(preparedQuery.release(), txAlloc->HolderFactory.GetFunctionRegistry());
NKikimr::NKqp::TQueryData::TPtr params = std::make_shared<NKikimr::NKqp::TQueryData>(txAlloc);
@@ -2270,24 +2193,6 @@ private:
externalTableDesc.SetContent(general.SerializeAsString());
}
- static void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescription& externaDataSourceDesc,
- const TString& name,
- const NYql::TCreateExternalDataSourceSettings& settings)
- {
- externaDataSourceDesc.SetName(name);
- externaDataSourceDesc.SetSourceType(settings.SourceType);
- externaDataSourceDesc.SetLocation(settings.Location);
- externaDataSourceDesc.SetInstallation(settings.Installation);
-
- if (settings.AuthMethod == "NONE") {
- externaDataSourceDesc.MutableAuth()->MutableNone();
- } else if (settings.AuthMethod == "SERVICE_ACCOUNT") {
- auto& sa = *externaDataSourceDesc.MutableAuth()->MutableServiceAccount();
- sa.SetId(settings.ServiceAccount.Id);
- sa.SetSecretName(settings.ServiceAccount.SecretName);
- }
- }
-
static void FillParameters(TQueryData::TPtr params, ::google::protobuf::Map<TBasicString<char>, Ydb::TypedValue>* output) {
if (!params) {
return;
diff --git a/ydb/core/kqp/gateway/ya.make b/ydb/core/kqp/gateway/ya.make
index 684bc256ec..ad479b21a9 100644
--- a/ydb/core/kqp/gateway/ya.make
+++ b/ydb/core/kqp/gateway/ya.make
@@ -15,6 +15,7 @@ PEERDIR(
ydb/core/kqp/query_data
ydb/core/kqp/gateway/actors
ydb/core/kqp/gateway/behaviour/tablestore
+ ydb/core/kqp/gateway/behaviour/external_data_source
ydb/library/yql/providers/result/expr_nodes
)
diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp
index 65707bb83c..b17d6d8d97 100644
--- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp
+++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp
@@ -640,35 +640,16 @@ public:
FORWARD_ENSURE_NO_PREPARE(DropExternalTable, cluster, settings);
}
- TFuture<TGenericResult> CreateExternalDataSource(const TString& cluster,
- const TCreateExternalDataSourceSettings& settings, bool createDir) override
- {
- FORWARD_ENSURE_NO_PREPARE(CreateExternalDataSource, cluster, settings, createDir);
- }
-
- TFuture<TGenericResult> AlterExternalDataSource(const TString& cluster,
- const TAlterExternalDataSourceSettings& settings) override
- {
- FORWARD_ENSURE_NO_PREPARE(AlterExternalDataSource, cluster, settings);
- }
-
- TFuture<TGenericResult> DropExternalDataSource(const TString& cluster,
- const TDropExternalDataSourceSettings& settings) override
- {
- FORWARD_ENSURE_NO_PREPARE(DropExternalDataSource, cluster, settings);
- }
-
TVector<TString> GetCollectedSchemeData() override {
return Gateway->GetCollectedSchemeData();
}
- TFuture<TExecuteLiteralResult> ExecuteLiteral(const TString& program,
- const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) override
+ TFuture<TExecuteLiteralResult> ExecuteLiteral(const TString& program,
+ const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) override
{
return Gateway->ExecuteLiteral(program, resultType, txAlloc);
}
-
private:
bool IsPrepare() const {
if (!SessionCtx) {
diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
index 3e8d8fd27d..ec0792a718 100644
--- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
@@ -179,35 +179,6 @@ namespace {
return dropGroupSettings;
}
- TString GetOrDefault(const TCreateObjectSettings& container, const TString& key, const TString& defaultValue = TString{}) {
- auto fValue = container.GetFeaturesExtractor().Extract(key);
- return fValue ? *fValue : defaultValue;
- }
-
- TCreateExternalDataSourceSettings ParseCreateExternalDataSourceSettings(const TCreateObjectSettings& settings) {
- TCreateExternalDataSourceSettings out;
- out.ExternalDataSource = settings.GetObjectId();
- out.SourceType = GetOrDefault(settings, "source_type");
- out.AuthMethod = GetOrDefault(settings, "auth_method");
- out.ServiceAccount.Id = GetOrDefault(settings, "service_account_id");
- out.ServiceAccount.SecretName = GetOrDefault(settings, "service_account_secret_name");
- out.Installation = GetOrDefault(settings, "installation");
- out.Location = GetOrDefault(settings, "location");
- return out;
- }
-
- TAlterExternalDataSourceSettings ParseAlterExternalDataSourceSettings(const TAlterObjectSettings& settings) {
- TAlterExternalDataSourceSettings out;
- out.ExternalDataSource = settings.GetObjectId();
- return out;
- }
-
- TDropExternalDataSourceSettings ParseDropExternalDataSourceSettings(const TDropObjectSettings& settings) {
- TDropExternalDataSourceSettings out;
- out.ExternalDataSource = settings.GetObjectId();
- return out;
- }
-
TCreateTableStoreSettings ParseCreateTableStoreSettings(TKiCreateTable create, const TTableSettings& settings) {
TCreateTableStoreSettings out;
out.TableStore = TString(create.Table());
@@ -564,7 +535,7 @@ private:
*SessionCtx->Query().QueryData->GetAllocState()->HolderFactory.GetFunctionRegistry(),
ctx, fakeReads);
- NKikimr::NMiniKQL::TProgramBuilder programBuilder(SessionCtx->Query().QueryData->GetAllocState()->TypeEnv,
+ NKikimr::NMiniKQL::TProgramBuilder programBuilder(SessionCtx->Query().QueryData->GetAllocState()->TypeEnv,
*SessionCtx->Query().QueryData->GetAllocState()->HolderFactory.GetFunctionRegistry());
TStringStream errorStream;
@@ -584,7 +555,7 @@ private:
NYson::TYsonWriter writer2((IOutputStream*)&out);
writer2.OnBeginMap();
writer2.OnKeyedItem("Data");
- writer2.OnRaw(ysonStream.Str());
+ writer2.OnRaw(ysonStream.Str());
writer2.OnEndMap();
return out.Str();
@@ -867,48 +838,6 @@ public:
using TBase::TBase;
};
-class TCreateExternalDataSourceTransformer: public TObjectModifierTransformer<TKiCreateObject, TCreateObjectSettings> {
-private:
- using TBase = TObjectModifierTransformer<TKiCreateObject, TCreateObjectSettings>;
-protected:
- virtual TFuture<IKikimrGateway::TGenericResult> DoExecute(const TString& cluster, const TCreateObjectSettings& settings) override {
- if (!SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) {
- return MakeErrorFuture<IKikimrGateway::TGenericResult>(std::make_exception_ptr(yexception() << "External data sources are disabled. Please contact your system administrator to enable it"));
- }
- return GetGateway()->CreateExternalDataSource(cluster, ParseCreateExternalDataSourceSettings(settings), true);
- }
-public:
- using TBase::TBase;
-};
-
-class TAlterExternalDataSourceTransformer: public TObjectModifierTransformer<TKiAlterObject, TAlterObjectSettings> {
-private:
- using TBase = TObjectModifierTransformer<TKiAlterObject, TAlterObjectSettings>;
-protected:
- virtual TFuture<IKikimrGateway::TGenericResult> DoExecute(const TString& cluster, const TAlterObjectSettings& settings) override {
- if (!SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) {
- return MakeErrorFuture<IKikimrGateway::TGenericResult>(std::make_exception_ptr(yexception() << "External data sources are disabled. Please contact your system administrator to enable it"));
- }
- return GetGateway()->AlterExternalDataSource(cluster, ParseAlterExternalDataSourceSettings(settings));
- }
-public:
- using TBase::TBase;
-};
-
-class TDropExternalDataSourceTransformer: public TObjectModifierTransformer<TKiDropObject, TDropObjectSettings> {
-private:
- using TBase = TObjectModifierTransformer<TKiDropObject, TDropObjectSettings>;
-protected:
- virtual TFuture<IKikimrGateway::TGenericResult> DoExecute(const TString& cluster, const TDropObjectSettings& settings) override {
- if (!SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) {
- return MakeErrorFuture<IKikimrGateway::TGenericResult>(std::make_exception_ptr(yexception() << "External data sources are disabled. Please contact your system administrator to enable it"));
- }
- return GetGateway()->DropExternalDataSource(cluster, ParseDropExternalDataSourceSettings(settings));
- }
-public:
- using TBase::TBase;
-};
-
class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKiSinkCallableExecutionTransformer> {
public:
TKiSinkCallableExecutionTransformer(
@@ -1774,21 +1703,15 @@ public:
}
if (auto kiObject = TMaybeNode<TKiCreateObject>(input)) {
- return kiObject.Cast().TypeId() == "EXTERNAL_DATA_SOURCE"
- ? TCreateExternalDataSourceTransformer("CREATE EXTERNAL DATA SOURCE", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx)
- : TCreateObjectTransformer("CREATE OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx);
+ return TCreateObjectTransformer("CREATE OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx);
}
if (auto kiObject = TMaybeNode<TKiAlterObject>(input)) {
- return kiObject.Cast().TypeId() == "EXTERNAL_DATA_SOURCE"
- ? TAlterExternalDataSourceTransformer("ALTER EXTERNAL DATA SOURCE", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx)
- : TAlterObjectTransformer("ALTER OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx);
+ return TAlterObjectTransformer("ALTER OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx);
}
if (auto kiObject = TMaybeNode<TKiDropObject>(input)) {
- return kiObject.Cast().TypeId() == "EXTERNAL_DATA_SOURCE"
- ? TDropExternalDataSourceTransformer("DROP EXTERNAL DATA SOURCE", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx)
- : TDropObjectTransformer("DROP OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx);
+ return TDropObjectTransformer("DROP OBJECT", Gateway, SessionCtx).Execute(kiObject.Cast(), input, ctx);
}
if (auto maybeCreateGroup = TMaybeNode<TKiCreateGroup>(input)) {
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h
index 60bbcff13e..f5076573d1 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway.h
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h
@@ -587,28 +587,6 @@ struct TDropExternalTableSettings {
TString ExternalTable;
};
-struct TCreateExternalDataSourceSettings {
- struct TServiceAccount {
- TString Id;
- TString SecretName;
- };
-
- TString ExternalDataSource;
- TString SourceType;
- TString Location;
- TString Installation;
- TString AuthMethod;
- TServiceAccount ServiceAccount;
-};
-
-struct TAlterExternalDataSourceSettings {
- TString ExternalDataSource;
-};
-
-struct TDropExternalDataSourceSettings {
- TString ExternalDataSource;
-};
-
struct TKikimrListPathItem {
TKikimrListPathItem(TString name, bool isDirectory) {
Name = name;
@@ -791,12 +769,6 @@ public:
virtual NThreading::TFuture<TGenericResult> DropExternalTable(const TString& cluster, const TDropExternalTableSettings& settings) = 0;
- virtual NThreading::TFuture<TGenericResult> CreateExternalDataSource(const TString& cluster, const TCreateExternalDataSourceSettings& settings, bool createDir) = 0;
-
- virtual NThreading::TFuture<TGenericResult> AlterExternalDataSource(const TString& cluster, const TAlterExternalDataSourceSettings& settings) = 0;
-
- virtual NThreading::TFuture<TGenericResult> DropExternalDataSource(const TString& cluster, const TDropExternalDataSourceSettings& settings) = 0;
-
virtual TVector<TString> GetCollectedSchemeData() = 0;
virtual NThreading::TFuture<TExecuteLiteralResult> ExecuteLiteral(const TString& program, const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) = 0;
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
index a390e09ac6..a96701e163 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
@@ -134,13 +134,12 @@ void TestDropTableCommon(TIntrusivePtr<IKikimrGateway> gateway) {
}
void TestCreateExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) {
- NYql::TCreateExternalDataSourceSettings settings;
- settings.ExternalDataSource = path;
- settings.SourceType = "ObjectStorage";
- settings.AuthMethod = "NONE";
- settings.Installation = "cloud";
-
- auto responseFuture = gateway->CreateExternalDataSource(TestCluster, settings, true);
+ TCreateObjectSettings settings("EXTERNAL_DATA_SOURCE", path, {
+ {"source_type", "ObjectStorage"},
+ {"auth_method", "NONE"},
+ {"installation", "cloud"}
+ });
+ auto responseFuture = gateway->CreateObject(TestCluster, settings);
responseFuture.Wait();
auto response = responseFuture.GetValue();
response.Issues().PrintTo(Cerr);
@@ -160,7 +159,6 @@ void TestCreateExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKik
void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path, bool fail = false) {
NYql::TCreateExternalTableSettings settings;
-
settings.ExternalTable = path;
settings.DataSourcePath = "/Root/f1/f2/external_data_source";
settings.Location = "/";
@@ -191,7 +189,6 @@ void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGa
}
void TestDropExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) {
-
auto responseFuture = gateway->DropExternalTable(TestCluster, TDropExternalTableSettings{.ExternalTable=path});
responseFuture.Wait();
auto response = responseFuture.GetValue();
@@ -205,7 +202,8 @@ void TestDropExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGate
}
void TestDropExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) {
- auto responseFuture = gateway->DropExternalDataSource(TestCluster, TDropExternalDataSourceSettings{.ExternalDataSource=path});
+ TDropObjectSettings settings("EXTERNAL_DATA_SOURCE", path, {});
+ auto responseFuture = gateway->DropObject(TestCluster, settings);
responseFuture.Wait();
auto response = responseFuture.GetValue();
response.Issues().PrintTo(Cerr);
@@ -241,12 +239,14 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
Y_UNIT_TEST(TestCreateExternalTable) {
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
+ kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
}
Y_UNIT_TEST(TestCreateSameExternalTable) {
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
+ kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table", true);
@@ -254,6 +254,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
Y_UNIT_TEST(TestDropExternalTable) {
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
+ kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
TestDropExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
@@ -261,6 +262,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
Y_UNIT_TEST(TestDropExternalDataSource) {
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
+ kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
TestDropExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
}
diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
index 5578d6a69f..05974c28bb 100644
--- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
+++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
@@ -4476,7 +4476,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
AUTH_METHOD="NONE"
);)";
auto result = session.ExecuteSchemeQuery(query).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::INTERNAL_ERROR);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "External data sources are disabled. Please contact your system administrator to enable it");
}
diff --git a/ydb/services/metadata/abstract/parsing.h b/ydb/services/metadata/abstract/parsing.h
index 44db7b66f1..433bd6828b 100644
--- a/ydb/services/metadata/abstract/parsing.h
+++ b/ydb/services/metadata/abstract/parsing.h
@@ -18,6 +18,14 @@ private:
TFeatures Features;
std::shared_ptr<TFeaturesExtractor> FeaturesExtractor;
public:
+ TObjectSettingsImpl() = default;
+
+ TObjectSettingsImpl(const TString& typeId, const TString& objectId, const TFeatures& features)
+ : TypeId(typeId)
+ , ObjectId(objectId)
+ , Features(features)
+ , FeaturesExtractor(std::make_shared<TFeaturesExtractor>(Features))
+ {}
TFeaturesExtractor& GetFeaturesExtractor() const {
Y_VERIFY(!!FeaturesExtractor);
diff --git a/ydb/services/metadata/manager/abstract.h b/ydb/services/metadata/manager/abstract.h
index 48b884f53c..b03cc8bbb2 100644
--- a/ydb/services/metadata/manager/abstract.h
+++ b/ydb/services/metadata/manager/abstract.h
@@ -14,6 +14,7 @@
#include <ydb/services/metadata/abstract/parsing.h>
#include <library/cpp/threading/future/core/future.h>
+#include <library/cpp/actors/core/actorsystem.h>
namespace NKikimr::NMetadata::NModifications {
@@ -62,6 +63,8 @@ public:
private:
YDB_ACCESSOR_DEF(std::optional<NACLib::TUserToken>, UserToken);
YDB_ACCESSOR_DEF(TString, Database);
+ using TActorSystemPtr = TActorSystem*;
+ YDB_ACCESSOR_DEF(TActorSystemPtr, ActorSystem);
};
class TInternalModificationContext {