aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhcpp <hcpp@ydb.tech>2023-02-08 13:20:50 +0300
committerhcpp <hcpp@ydb.tech>2023-02-08 13:20:50 +0300
commit6bf6928eda1dd0bcabb3aac0000b72aedd29d44e (patch)
treebbff709ccdc617317577432989f871a11f58a5ee
parent276e61af9d8785166ff14771a61e0acd8527b282 (diff)
downloadydb-6bf6928eda1dd0bcabb3aac0000b72aedd29d44e.tar.gz
external table has been added to ss
-rw-r--r--ydb/core/protos/counters_schemeshard.proto9
-rw-r--r--ydb/core/protos/flat_scheme_op.proto21
-rw-r--r--ydb/core/tx/scheme_board/cache.cpp17
-rw-r--r--ydb/core/tx/scheme_cache/scheme_cache.h7
-rw-r--r--ydb/core/tx/schemeshard/CMakeLists.darwin.txt4
-rw-r--r--ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt4
-rw-r--r--ydb/core/tx/schemeshard/CMakeLists.linux.txt4
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__init.cpp53
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation.cpp21
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp344
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp209
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp18
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h6
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_part.h8
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp15
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.cpp63
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.h5
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.h11
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path.cpp13
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path_describer.cpp37
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path_describer.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path_element.cpp4
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path_element.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_schema.h16
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_tx_infly.h14
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table.cpp237
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt82
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt85
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt87
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt15
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table_reboots.cpp214
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt82
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt85
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt87
-rw-r--r--ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt15
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/helpers.cpp5
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/helpers.h6
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp7
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.h1
-rw-r--r--ydb/core/tx/tx_proxy/schemereq.cpp13
-rw-r--r--ydb/core/viewer/browse.h2
-rw-r--r--ydb/core/viewer/protos/viewer.proto1
-rw-r--r--ydb/public/lib/deprecated/kicli/kicli.h1
-rw-r--r--ydb/public/lib/deprecated/kicli/schema.cpp5
-rw-r--r--ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema72
47 files changed, 1993 insertions, 16 deletions
diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto
index 9c3ed4d879..cf5067a51c 100644
--- a/ydb/core/protos/counters_schemeshard.proto
+++ b/ydb/core/protos/counters_schemeshard.proto
@@ -173,6 +173,11 @@ enum ESimpleCounters {
COUNTER_IN_FLIGHT_OPS_TxAlterExtSubDomainCreateHive = 140 [(CounterOpts) = {Name: "InFlightOps/AlterExtSubDomainCreateHive"}];
COUNTER_IN_FLIGHT_OPS_TxAlterCdcStreamAtTableDropSnapshot = 141 [(CounterOpts) = {Name: "InFlightOps/AlterCdcStreamAtTableDropSnapshot"}];
COUNTER_IN_FLIGHT_OPS_TxDropCdcStreamAtTableDropSnapshot = 142 [(CounterOpts) = {Name: "InFlightOps/DropCdcStreamAtTableDropSnapshot"}];
+
+ COUNTER_EXTERNAL_TABLE_COUNT = 143 [(CounterOpts) = {Name: "ExternalTableCount"}];
+ COUNTER_IN_FLIGHT_OPS_TxCreateExternalTable = 144 [(CounterOpts) = {Name: "InFlightOps/CreateExternalTable"}];
+ COUNTER_IN_FLIGHT_OPS_TxDropExternalTable = 145 [(CounterOpts) = {Name: "InFlightOps/DropExternalTable"}];
+ COUNTER_IN_FLIGHT_OPS_TxAlterExternalTable = 146 [(CounterOpts) = {Name: "InFlightOps/AlterExternalTable"}];
}
enum ECumulativeCounters {
@@ -279,6 +284,10 @@ enum ECumulativeCounters {
COUNTER_FINISHED_OPS_TxAlterExtSubDomainCreateHive = 85 [(CounterOpts) = {Name: "FinishedOps/AlterExtSubDomainCreateHive"}];
COUNTER_FINISHED_OPS_TxAlterCdcStreamAtTableDropSnapshot = 86 [(CounterOpts) = {Name: "FinishedOps/AlterCdcStreamAtTableDropSnapshot"}];
COUNTER_FINISHED_OPS_TxDropCdcStreamAtTableDropSnapshot = 87 [(CounterOpts) = {Name: "FinishedOps/DropCdcStreamAtTableDropSnapshot"}];
+
+ COUNTER_FINISHED_OPS_TxCreateExternalTable = 88 [(CounterOpts) = {Name: "FinishedOps/CreateExternalTable"}];
+ COUNTER_FINISHED_OPS_TxDropExternalTable = 89 [(CounterOpts) = {Name: "FinishedOps/DropExternalTable"}];
+ COUNTER_FINISHED_OPS_TxAlterExternalTable = 90 [(CounterOpts) = {Name: "FinishedOps/AlterExternalTable"}];
}
enum EPercentileCounters {
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 81b930edb5..826e62b31f 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -1297,6 +1297,11 @@ enum EOperationType {
// AlterExtSubDomain suboperations
ESchemeOpAlterExtSubDomainCreateHive = 85;
+
+ // External Table
+ ESchemeOpCreateExternalTable = 86;
+ ESchemeOpDropExternalTable = 87;
+ ESchemeOpAlterExternalTable = 88;
}
message TApplyIf {
@@ -1405,6 +1410,8 @@ message TModifyScheme {
optional TPersQueueGroupAllocate AllocatePersQueueGroup = 56;
optional TPersQueueGroupDeallocate DeallocatePersQueueGroup = 57;
+
+ optional TExternalTableDescription CreateExternalTable = 58;
}
// "Script", used by client to parse text files with multiple DDL commands
@@ -1460,6 +1467,7 @@ enum EPathType {
EPathTypeSequence = 15;
EPathTypeReplication = 16;
EPathTypeBlobDepot = 17;
+ EPathTypeExternalTable = 18;
}
enum EPathSubType {
@@ -1511,6 +1519,7 @@ message TPathVersion {
optional uint64 SequenceVersion = 24;
optional uint64 ReplicationVersion = 25;
optional uint64 BlobDepotVersion = 26;
+ optional uint64 ExternalTableVersion = 27;
}
// Describes single path
@@ -1596,6 +1605,7 @@ message TPathDescription {
optional TSequenceDescription SequenceDescription = 24;
optional TReplicationDescription ReplicationDescription = 25;
optional TBlobDepotDescription BlobDepotDescription = 26;
+ optional TExternalTableDescription ExternalTableDescription = 27;
}
// For persisting AlterTable Tx description in Schemeshard internal DB
@@ -1639,3 +1649,14 @@ message TResourceProfile {
// Resource Broker task type for large transactions.
optional string LargeTxTaskType = 12 [default = "transaction"];
}
+
+message TExternalTableDescription {
+ optional string Name = 1;
+ optional NKikimrProto.TPathID PathId = 2;
+ optional uint64 Version = 3;
+ optional string SourceType = 4;
+ optional string DataSourcePath = 5;
+ optional string Location = 6;
+ repeated TColumnDescription Columns = 7;
+ optional bytes Content = 8;
+}
diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp
index 8955be17b5..b0c7b7fe85 100644
--- a/ydb/core/tx/scheme_board/cache.cpp
+++ b/ydb/core/tx/scheme_board/cache.cpp
@@ -736,6 +736,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
SequenceInfo.Drop();
ReplicationInfo.Drop();
BlobDepotInfo.Drop();
+ ExternalTableInfo.Drop();
}
void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) {
@@ -1148,8 +1149,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
root.WriteKey(#name).UnsafeWriteValue(ProtoJsonString(name->Description)); \
}
- DESCRIPTION_PART(DomainDescription)
- DESCRIPTION_PART(RtmrVolumeInfo)
+ DESCRIPTION_PART(DomainDescription);
+ DESCRIPTION_PART(RtmrVolumeInfo);
DESCRIPTION_PART(KesusInfo);
DESCRIPTION_PART(SolomonVolumeInfo);
DESCRIPTION_PART(PQGroupInfo);
@@ -1159,6 +1160,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
DESCRIPTION_PART(SequenceInfo);
DESCRIPTION_PART(ReplicationInfo);
DESCRIPTION_PART(BlobDepotInfo);
+ DESCRIPTION_PART(ExternalTableInfo);
#undef DESCRIPTION_PART
@@ -1465,6 +1467,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
Kind = TNavigate::KindBlobDepot;
FillInfo(Kind, BlobDepotInfo, std::move(*pathDesc.MutableBlobDepotDescription()));
break;
+ case NKikimrSchemeOp::EPathTypeExternalTable:
+ Kind = TNavigate::KindExternalTable;
+ FillInfo(Kind, ExternalTableInfo, std::move(*pathDesc.MutableExternalTableDescription()));
+ break;
case NKikimrSchemeOp::EPathTypeInvalid:
Y_VERIFY_DEBUG(false, "Invalid path type");
break;
@@ -1522,6 +1528,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
case NKikimrSchemeOp::EPathTypeBlobDepot:
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindBlobDepot);
break;
+ case NKikimrSchemeOp::EPathTypeExternalTable:
+ ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindExternalTable);
+ break;
case NKikimrSchemeOp::EPathTypeTableIndex:
case NKikimrSchemeOp::EPathTypeInvalid:
Y_VERIFY_DEBUG(false, "Invalid path type");
@@ -1730,6 +1739,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
entry.SequenceInfo = SequenceInfo;
entry.ReplicationInfo = ReplicationInfo;
entry.BlobDepotInfo = BlobDepotInfo;
+ entry.ExternalTableInfo = ExternalTableInfo;
}
bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry,
@@ -2004,6 +2014,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
// BlobDepot specific
TIntrusivePtr<TNavigate::TBlobDepotInfo> BlobDepotInfo;
+ // ExternalTable specific
+ TIntrusivePtr<TNavigate::TExternalTableInfo> ExternalTableInfo;
+
}; // TCacheItem
struct TMerger {
diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h
index 4786dd0887..913cc74f7e 100644
--- a/ydb/core/tx/scheme_cache/scheme_cache.h
+++ b/ydb/core/tx/scheme_cache/scheme_cache.h
@@ -129,6 +129,7 @@ struct TSchemeCacheNavigate {
KindSequence = 14,
KindReplication = 15,
KindBlobDepot = 16,
+ KindExternalTable = 17,
};
struct TListNodeEntry : public TAtomicRefCount<TListNodeEntry> {
@@ -211,6 +212,11 @@ struct TSchemeCacheNavigate {
NKikimrSchemeOp::TBlobDepotDescription Description;
};
+ struct TExternalTableInfo : public TAtomicRefCount<TExternalTableInfo> {
+ EKind Kind = KindUnknown;
+ NKikimrSchemeOp::TExternalTableDescription Description;
+ };
+
struct TEntry {
enum class ERequestType : ui8 {
ByPath,
@@ -256,6 +262,7 @@ struct TSchemeCacheNavigate {
TIntrusiveConstPtr<TSequenceInfo> SequenceInfo;
TIntrusiveConstPtr<TReplicationInfo> ReplicationInfo;
TIntrusiveConstPtr<TBlobDepotInfo> BlobDepotInfo;
+ TIntrusiveConstPtr<TExternalTableInfo> ExternalTableInfo;
TString ToString() const;
TString ToString(const NScheme::TTypeRegistry& typeRegistry) const;
diff --git a/ydb/core/tx/schemeshard/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/CMakeLists.darwin.txt
index a6d51a786d..9db71ca37f 100644
--- a/ydb/core/tx/schemeshard/CMakeLists.darwin.txt
+++ b/ydb/core/tx/schemeshard/CMakeLists.darwin.txt
@@ -17,6 +17,8 @@ add_subdirectory(ut_cdc_stream_reboots)
add_subdirectory(ut_compaction)
add_subdirectory(ut_export)
add_subdirectory(ut_export_reboots_s3)
+add_subdirectory(ut_external_table)
+add_subdirectory(ut_external_table_reboots)
add_subdirectory(ut_extsubdomain)
add_subdirectory(ut_extsubdomain_reboots)
add_subdirectory(ut_filestore_reboots)
@@ -136,6 +138,7 @@ target_sources(core-tx-schemeshard PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_fs.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp
@@ -153,6 +156,7 @@ target_sources(core-tx-schemeshard PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_extsubdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_fs.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp
diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt
index 5ed31fe71d..49c3a0454e 100644
--- a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt
@@ -17,6 +17,8 @@ add_subdirectory(ut_cdc_stream_reboots)
add_subdirectory(ut_compaction)
add_subdirectory(ut_export)
add_subdirectory(ut_export_reboots_s3)
+add_subdirectory(ut_external_table)
+add_subdirectory(ut_external_table_reboots)
add_subdirectory(ut_extsubdomain)
add_subdirectory(ut_extsubdomain_reboots)
add_subdirectory(ut_filestore_reboots)
@@ -137,6 +139,7 @@ target_sources(core-tx-schemeshard PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_fs.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp
@@ -154,6 +157,7 @@ target_sources(core-tx-schemeshard PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_extsubdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_fs.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp
diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/CMakeLists.linux.txt
index 5ed31fe71d..49c3a0454e 100644
--- a/ydb/core/tx/schemeshard/CMakeLists.linux.txt
+++ b/ydb/core/tx/schemeshard/CMakeLists.linux.txt
@@ -17,6 +17,8 @@ add_subdirectory(ut_cdc_stream_reboots)
add_subdirectory(ut_compaction)
add_subdirectory(ut_export)
add_subdirectory(ut_export_reboots_s3)
+add_subdirectory(ut_external_table)
+add_subdirectory(ut_external_table_reboots)
add_subdirectory(ut_extsubdomain)
add_subdirectory(ut_extsubdomain_reboots)
add_subdirectory(ut_filestore_reboots)
@@ -137,6 +139,7 @@ target_sources(core-tx-schemeshard PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_fs.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp
@@ -154,6 +157,7 @@ target_sources(core-tx-schemeshard PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_extsubdomain.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_fs.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
index 9b3ea10a73..d0a0736874 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -1877,6 +1877,30 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
}
+ // Read External Tables
+ {
+ auto rowset = db.Table<Schema::ExternalTable>().Range().Select();
+ if (!rowset.IsReady())
+ return false;
+
+ while (!rowset.EndOfSet()) {
+ TOwnerId ownerPathId = rowset.GetValue<Schema::ExternalTable::OwnerPathId>();
+ TLocalPathId localPathId = rowset.GetValue<Schema::ExternalTable::LocalPathId>();
+ TPathId pathId(ownerPathId, localPathId);
+
+ auto& externalTable = Self->ExternalTables[pathId] = new TExternalTableInfo();
+ externalTable->SourceType = rowset.GetValue<Schema::ExternalTable::SourceType>();
+ externalTable->DataSourcePath = rowset.GetValue<Schema::ExternalTable::DataSourcePath>();
+ externalTable->Location = rowset.GetValue<Schema::ExternalTable::Location>();
+ externalTable->AlterVersion = rowset.GetValue<Schema::ExternalTable::AlterVersion>();
+ externalTable->Content = rowset.GetValue<Schema::ExternalTable::Content>();
+ Self->IncrementPathDbRefCount(pathId);
+
+ if (!rowset.Next())
+ return false;
+ }
+ }
+
// Read table columns
{
TColumnRows columnRows;
@@ -1903,13 +1927,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
auto notNull = std::get<10>(rec);
Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId);
- Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable(), "Path is not a table, pathId: " << pathId);
- Y_VERIFY_S(Self->Tables.FindPtr(pathId), "Table doesn't exist, pathId: " << pathId);
-
- TTableInfo::TPtr tableInfo = Self->Tables[pathId];
- Y_VERIFY_S(colId < tableInfo->NextColumnId, "Column id should be less than NextColId"
- << ", columnId: " << colId
- << ", NextColId: " << tableInfo->NextColumnId);
+ Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable() || Self->PathsById.at(pathId)->IsExternalTable(), "Path is not a table or external table, pathId: " << pathId);
+ Y_VERIFY_S(Self->Tables.FindPtr(pathId) || Self->ExternalTables.FindPtr(pathId), "Table or external table don't exist, pathId: " << pathId);
TTableInfo::TColumn colInfo(colName, colId, typeInfo);
colInfo.KeyOrder = keyOrder;
@@ -1920,11 +1939,21 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
colInfo.DefaultValue = defaultValue;
colInfo.NotNull = notNull;
- tableInfo->Columns[colId] = colInfo;
+ if (auto it = Self->Tables.find(pathId); it != Self->Tables.end()) {
+ TTableInfo::TPtr tableInfo = it->second;
+ Y_VERIFY_S(colId < tableInfo->NextColumnId, "Column id should be less than NextColId"
+ << ", columnId: " << colId
+ << ", NextColId: " << tableInfo->NextColumnId);
- if (colInfo.KeyOrder != (ui32)-1) {
- tableInfo->KeyColumnIds.resize(Max<ui32>(tableInfo->KeyColumnIds.size(), colInfo.KeyOrder + 1));
- tableInfo->KeyColumnIds[colInfo.KeyOrder] = colId;
+ tableInfo->Columns[colId] = colInfo;
+
+ if (colInfo.KeyOrder != (ui32)-1) {
+ tableInfo->KeyColumnIds.resize(Max<ui32>(tableInfo->KeyColumnIds.size(), colInfo.KeyOrder + 1));
+ tableInfo->KeyColumnIds[colInfo.KeyOrder] = colId;
+ }
+ } else if (auto it = Self->ExternalTables.find(pathId); it != Self->ExternalTables.end()) {
+ TExternalTableInfo::TPtr externalTableInfo = it->second;
+ externalTableInfo->Columns[colId] = colInfo;
}
}
}
@@ -3945,6 +3974,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
Self->TabletCounters->Simple()[COUNTER_SEQUENCE_COUNT].Add(1);
} else if (path->IsReplication()) {
Self->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(1);
+ } else if (path->IsExternalTable()) {
+ Self->TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Add(1);
}
path->ApplySpecialAttributes();
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp
index bb83d34e90..4d7fccde2b 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp
@@ -666,6 +666,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable:
targetName = tx.GetCreateColumnTable().GetName();
break;
+ case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable:
+ targetName = tx.GetCreateExternalTable().GetName();
+ break;
default:
result.Transactions.push_back(tx);
return result;
@@ -760,6 +763,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable:
create.MutableCreateColumnTable()->SetName(name);
break;
+ case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable:
+ create.MutableCreateExternalTable()->SetName(name);
+ break;
default:
Y_UNREACHABLE();
}
@@ -1004,7 +1010,12 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta
return CreateAlterBlobDepot(NextPartId(), txState);
case TTxState::ETxType::TxDropBlobDepot:
return CreateDropBlobDepot(NextPartId(), txState);
-
+ case TTxState::ETxType::TxCreateExternalTable:
+ return CreateNewExternalTable(NextPartId(), txState);
+ case TTxState::ETxType::TxDropExternalTable:
+ return CreateDropExternalTable(NextPartId(), txState);
+ case TTxState::ETxType::TxAlterExternalTable:
+ Y_FAIL("TODO: implement");
case TTxState::ETxType::TxInvalid:
Y_UNREACHABLE();
}
@@ -1205,6 +1216,14 @@ ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationTyp
return CreateAlterBlobDepot(NextPartId(), tx);
case NKikimrSchemeOp::EOperationType::ESchemeOpDropBlobDepot:
return CreateDropBlobDepot(NextPartId(), tx);
+
+ // ExternalTable
+ case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable:
+ return CreateNewExternalTable(NextPartId(), tx);
+ case NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable:
+ return CreateDropExternalTable(NextPartId(), tx);
+ case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExternalTable:
+ Y_FAIL("TODO: implement");
}
Y_UNREACHABLE();
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp
new file mode 100644
index 0000000000..7621265af9
--- /dev/null
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp
@@ -0,0 +1,344 @@
+#include "schemeshard__operation_part.h"
+#include "schemeshard__operation_common.h"
+#include "schemeshard_impl.h"
+
+#include <ydb/core/base/subdomain.h>
+
+#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream)
+#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream)
+
+
+namespace {
+
+using namespace NKikimr;
+using namespace NSchemeShard;
+
+bool IsAllowedType(ui32 typeId) {
+ if (!NScheme::NTypeIds::IsYqlType(typeId)) {
+ return false;
+ }
+
+ switch (typeId) {
+ case NYql::NProto::Bool:
+ case NYql::NProto::Interval:
+ case NYql::NProto::Decimal:
+ case NYql::NProto::DyNumber:
+ return false;
+ default:
+ break;
+ }
+ return true;
+}
+
+TExternalTableInfo::TPtr CreateExternalTable(const NKikimrSchemeOp::TExternalTableDescription& desc, TString& errStr) {
+ TExternalTableInfo::TPtr externalTableInfo = new TExternalTableInfo;
+ const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry;
+
+ externalTableInfo->DataSourcePath = desc.GetDataSourcePath();
+ externalTableInfo->Location = desc.GetLocation();
+ externalTableInfo->AlterVersion = 1;
+ externalTableInfo->Content = desc.GetContent();
+
+ uint64_t nextColumnId = 1;
+ for (const auto& col : desc.GetColumns()) {
+ TString colName = col.GetName();
+
+ if (!colName) {
+ errStr = "Columns cannot have an empty name";
+ return nullptr;
+ }
+
+ if (col.HasTypeId()) {
+ errStr = TStringBuilder() << "Cannot set TypeId for column '" << colName << "', use Type";
+ return nullptr;
+ }
+
+ if (!col.HasType()) {
+ errStr = TStringBuilder() << "Missing Type for column '" << colName << "'";
+ return nullptr;
+ }
+
+ auto typeName = NMiniKQL::AdaptLegacyYqlType(col.GetType());
+ const NScheme::IType* type = typeRegistry->GetType(typeName);
+
+ if (!type || !IsAllowedType(type->GetTypeId())) {
+ errStr = TStringBuilder()
+ << "Type '" << col.GetType() << "' specified for column '" << colName << "' is not supported";
+ return nullptr;
+ }
+
+ NScheme::TTypeInfo typeInfo;
+ if (type) {
+ // Only allow YQL types
+ if (!NScheme::NTypeIds::IsYqlType(type->GetTypeId())) {
+ errStr = Sprintf("Type '%s' specified for column '%s' is no longer supported", col.GetType().data(), colName.data());
+ return nullptr;
+ }
+ typeInfo = NScheme::TTypeInfo(type->GetTypeId());
+ } else {
+ auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName);
+ if (!typeDesc) {
+ errStr = Sprintf("Type '%s' specified for column '%s' is not supported by storage", col.GetType().data(), colName.data());
+ return nullptr;
+ }
+ typeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc);
+ }
+
+ ui32 colId = col.HasId() ? col.GetId() : nextColumnId;
+ if (externalTableInfo->Columns.contains(colId)) {
+ errStr = Sprintf("Duplicate column id: %" PRIu32, colId);
+ return nullptr;
+ }
+
+ nextColumnId = colId + 1 > nextColumnId ? colId + 1 : nextColumnId;
+
+ TTableInfo::TColumn& column = externalTableInfo->Columns[colId];
+ column = TTableInfo::TColumn(colName, colId, typeInfo);
+ column.NotNull = col.GetNotNull();
+ }
+
+ return externalTableInfo;
+}
+
+class TPropose: public TSubOperationState {
+private:
+ TOperationId OperationId;
+
+ TString DebugHint() const override {
+ return TStringBuilder()
+ << "TCreateExternalTable TPropose"
+ << ", operationId: " << OperationId;
+ }
+
+public:
+ TPropose(TOperationId id)
+ : OperationId(id)
+ {
+ }
+
+ bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override {
+ const TStepId step = TStepId(ev->Get()->StepId);
+
+ LOG_I(DebugHint() << "HandleReply TEvOperationPlan"
+ << ": step# " << step);
+
+ TTxState* txState = context.SS->FindTx(OperationId);
+ Y_VERIFY(txState);
+ Y_VERIFY(txState->TxType == TTxState::TxCreateExternalTable);
+
+ auto pathId = txState->TargetPathId;
+ auto path = TPath::Init(pathId, context.SS);
+ TPathElement::TPtr pathPtr = context.SS->PathsById.at(pathId);
+
+ context.SS->TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Add(1);
+
+ NIceDb::TNiceDb db(context.GetDB());
+
+ path->StepCreated = step;
+ context.SS->PersistCreateStep(db, pathId, step);
+
+ IncParentDirAlterVersionWithRepublish(OperationId, path, context);
+
+ context.SS->ClearDescribePathCaches(pathPtr);
+ context.OnComplete.PublishToSchemeBoard(OperationId, pathId);
+
+ context.SS->ChangeTxState(db, OperationId, TTxState::Done);
+ return true;
+ }
+
+ bool ProgressState(TOperationContext& context) override {
+ LOG_I(DebugHint() << "ProgressState");
+
+ TTxState* txState = context.SS->FindTx(OperationId);
+ Y_VERIFY(txState);
+ Y_VERIFY(txState->TxType == TTxState::TxCreateExternalTable);
+
+ context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0));
+ return false;
+ }
+};
+
+
+class TCreateExternalTable: public TSubOperation {
+ static TTxState::ETxState NextState() {
+ return TTxState::Propose;
+ }
+
+ TTxState::ETxState NextState(TTxState::ETxState state) const override {
+ switch (state) {
+ case TTxState::Waiting:
+ case TTxState::Propose:
+ return TTxState::Done;
+ default:
+ return TTxState::Invalid;
+ }
+ }
+
+ TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override {
+ switch (state) {
+ case TTxState::Waiting:
+ case TTxState::Propose:
+ return MakeHolder<TPropose>(OperationId);
+ case TTxState::Done:
+ return MakeHolder<TDone>(OperationId);
+ default:
+ return nullptr;
+ }
+ }
+
+public:
+ using TSubOperation::TSubOperation;
+
+ THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override {
+ const auto ssId = context.SS->SelfTabletId();
+
+ const auto acceptExisted = !Transaction.GetFailOnExist();
+ const TString& parentPathStr = Transaction.GetWorkingDir();
+ const auto& externalTableDescription = Transaction.GetCreateExternalTable();
+ const TString& name = externalTableDescription.GetName();
+
+
+ LOG_N("TCreateExternalTable Propose"
+ << ": opId# " << OperationId
+ << ", path# " << parentPathStr << "/" << name);
+
+ auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId));
+
+ NSchemeShard::TPath parentPath = NSchemeShard::TPath::Resolve(parentPathStr, context.SS);
+ {
+ NSchemeShard::TPath::TChecker checks = parentPath.Check();
+ checks
+ .NotUnderDomainUpgrade()
+ .IsAtLocalSchemeShard()
+ .IsResolved()
+ .NotDeleted()
+ .NotUnderDeleting()
+ .IsCommonSensePath()
+ .IsLikeDirectory();
+
+ if (!checks) {
+ result->SetError(checks.GetStatus(), checks.GetError());
+ return result;
+ }
+ }
+
+ const TString acl = Transaction.GetModifyACL().GetDiffACL();
+
+ NSchemeShard::TPath dstPath = parentPath.Child(name);
+ {
+ NSchemeShard::TPath::TChecker checks = dstPath.Check();
+ checks.IsAtLocalSchemeShard();
+ if (dstPath.IsResolved()) {
+ checks
+ .IsResolved()
+ .NotUnderDeleting()
+ .FailOnExist(TPathElement::EPathType::EPathTypeExternalTable, acceptExisted);
+ } else {
+ checks
+ .NotEmpty()
+ .NotResolved();
+ }
+
+ if (checks) {
+ checks
+ .IsValidLeafName()
+ .DepthLimit()
+ .PathsLimit()
+ .DirChildrenLimit()
+ .IsValidACL(acl);
+ }
+
+ if (!checks) {
+ result->SetError(checks.GetStatus(), checks.GetError());
+ if (dstPath.IsResolved()) {
+ result->SetPathCreateTxId(ui64(dstPath.Base()->CreateTxId));
+ result->SetPathId(dstPath.Base()->PathId.LocalPathId);
+ }
+ return result;
+ }
+ }
+
+ TString errStr;
+ if (!context.SS->CheckApplyIf(Transaction, errStr)) {
+ result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr);
+ return result;
+ }
+
+ TExternalTableInfo::TPtr externalTableInfo = CreateExternalTable(externalTableDescription, errStr);
+ if (!externalTableInfo) {
+ result->SetError(NKikimrScheme::StatusSchemeError, errStr);
+ return result;
+ }
+
+ dstPath.MaterializeLeaf(owner);
+ result->SetPathId(dstPath.Base()->PathId.LocalPathId);
+
+ TPathElement::TPtr externalTable = dstPath.Base();
+ externalTable->CreateTxId = OperationId.GetTxId();
+ externalTable->LastTxId = OperationId.GetTxId();
+ externalTable->PathState = TPathElement::EPathState::EPathStateCreate;
+ externalTable->PathType = TPathElement::EPathType::EPathTypeExternalTable;
+
+ TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateExternalTable, externalTable->PathId);
+ txState.Shards.clear();
+
+ NIceDb::TNiceDb db(context.GetDB());
+
+ if (parentPath.Base()->HasActiveChanges()) {
+ TTxId parentTxId = parentPath.Base()->PlannedToCreate() ? parentPath.Base()->CreateTxId : parentPath.Base()->LastTxId;
+ context.OnComplete.Dependence(parentTxId, OperationId.GetTxId());
+ }
+
+ context.SS->ChangeTxState(db, OperationId, TTxState::Propose);
+ context.OnComplete.ActivateTx(OperationId);
+
+ context.SS->ExternalTables[externalTable->PathId] = externalTableInfo;
+ context.SS->IncrementPathDbRefCount(externalTable->PathId);
+
+ context.SS->PersistPath(db, externalTable->PathId);
+
+ if (!acl.empty()) {
+ externalTable->ApplyACL(acl);
+ context.SS->PersistACL(db, externalTable);
+ }
+
+ context.SS->PersistExternalTable(db, externalTable->PathId, externalTableInfo);
+ context.SS->PersistTxState(db, OperationId);
+
+ IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, dstPath, context.SS, context.OnComplete);
+
+ dstPath.DomainInfo()->IncPathsInside();
+ parentPath.Base()->IncAliveChildren();
+
+ SetState(NextState());
+ return result;
+ }
+
+ void AbortPropose(TOperationContext& context) override {
+ LOG_N("TCreateExternalTable AbortPropose"
+ << ": opId# " << OperationId);
+ Y_FAIL("no AbortPropose for TCreateExternalTable");
+ }
+
+ void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override {
+ LOG_N("TCreateExternalTable AbortUnsafe"
+ << ": opId# " << OperationId
+ << ", txId# " << forceDropTxId);
+ context.OnComplete.DoneOperation(OperationId);
+ }
+};
+
+}
+
+namespace NKikimr::NSchemeShard {
+
+ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, const TTxTransaction& tx) {
+ return MakeSubOperation<TCreateExternalTable>(id, tx);
+}
+
+ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, TTxState::ETxState state) {
+ Y_VERIFY(state != TTxState::Invalid);
+ return MakeSubOperation<TCreateExternalTable>(id, state);
+}
+
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp
new file mode 100644
index 0000000000..cea1bf6ccb
--- /dev/null
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp
@@ -0,0 +1,209 @@
+#include "schemeshard__operation_common.h"
+#include "schemeshard__operation_part.h"
+#include "schemeshard_impl.h"
+
+#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream)
+#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream)
+
+namespace {
+
+using namespace NKikimr;
+using namespace NSchemeShard;
+
+class TPropose: public TSubOperationState {
+ TString DebugHint() const override {
+ return TStringBuilder()
+ << "TDropExternalTable TPropose"
+ << " opId# " << OperationId << " ";
+ }
+
+public:
+ explicit TPropose(TOperationId id)
+ : OperationId(id)
+ { }
+
+ bool ProgressState(TOperationContext& context) override {
+ LOG_I(DebugHint() << "ProgressState");
+
+ const auto* txState = context.SS->FindTx(OperationId);
+ Y_VERIFY(txState);
+ Y_VERIFY(txState->TxType == TTxState::TxDropExternalTable);
+
+ context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0));
+ return false;
+ }
+
+ bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override {
+ const auto step = TStepId(ev->Get()->StepId);
+
+ LOG_I(DebugHint() << "HandleReply TEvOperationPlan"
+ << ": step# " << step);
+
+ NIceDb::TNiceDb db(context.GetDB());
+
+ TTxState* txState = context.SS->FindTx(OperationId);
+ Y_VERIFY(txState);
+ TPathId pathId = txState->TargetPathId;
+ auto path = context.SS->PathsById.at(pathId);
+ auto parentDir = context.SS->PathsById.at(path->ParentPathId);
+
+ Y_VERIFY(!path->Dropped());
+ path->SetDropped(step, OperationId.GetTxId());
+ context.SS->PersistDropStep(db, pathId, step, OperationId);
+ auto domainInfo = context.SS->ResolveDomainInfo(pathId);
+ domainInfo->DecPathsInside();
+ parentDir->DecAliveChildren();
+
+ context.SS->TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Sub(1);
+ context.SS->PersistRemoveExternalTable(db, pathId);
+
+ ++parentDir->DirAlterVersion;
+ context.SS->PersistPathDirAlterVersion(db, parentDir);
+ context.SS->ClearDescribePathCaches(parentDir);
+ context.SS->ClearDescribePathCaches(path);
+
+ if (!context.SS->DisablePublicationsOfDropping) {
+ context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId);
+ context.OnComplete.PublishToSchemeBoard(OperationId, pathId);
+ }
+
+ context.SS->ChangeTxState(db, OperationId, TTxState::Done);
+
+ return true;
+ }
+
+private:
+ const TOperationId OperationId;
+
+}; // TPropose
+
+class TDropExternalTable: public TSubOperation {
+ TTxState::ETxState NextState() const {
+ return TTxState::Propose;
+ }
+
+ TTxState::ETxState NextState(TTxState::ETxState state) const override {
+ switch (state) {
+ case TTxState::Propose:
+ return TTxState::Done;
+ default:
+ return TTxState::Invalid;
+ }
+ }
+
+ TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override {
+ switch (state) {
+ case TTxState::Propose:
+ return MakeHolder<TPropose>(OperationId);
+ case TTxState::Done:
+ return MakeHolder<TDone>(OperationId);
+ default:
+ return nullptr;
+ }
+ }
+
+public:
+ using TSubOperation::TSubOperation;
+
+ THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override {
+ const ui64 ssId = context.SS->TabletID();
+ const auto& drop = Transaction.GetDrop();
+
+ const TString& workingDir = Transaction.GetWorkingDir();
+ const TString& name = drop.GetName();
+
+ LOG_N("TDropExternalTable Propose"
+ << ": opId# " << OperationId
+ << ", path# " << workingDir << "/" << name);
+
+ auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ssId);
+
+ TPath path = drop.HasId()
+ ? TPath::Init(context.SS->MakeLocalId(drop.GetId()), context.SS)
+ : TPath::Resolve(workingDir, context.SS).Dive(name);
+ {
+ TPath::TChecker checks = path.Check();
+ checks
+ .NotEmpty()
+ .NotUnderDomainUpgrade()
+ .IsAtLocalSchemeShard()
+ .IsResolved()
+ .NotDeleted()
+ .NotUnderDeleting()
+ .IsExternalTable()
+ .NotUnderOperation()
+ .IsCommonSensePath();
+
+ if (!checks) {
+ result->SetError(checks.GetStatus(), checks.GetError());
+ if (path.IsResolved() && path.Base()->IsExternalTable() && (path.Base()->PlannedToDrop() || path.Base()->Dropped())) {
+ result->SetPathDropTxId(ui64(path.Base()->DropTxId));
+ result->SetPathId(path.Base()->PathId.LocalPathId);
+ }
+ return result;
+ }
+ }
+
+ TString errStr;
+ if (!context.SS->CheckApplyIf(Transaction, errStr)) {
+ result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr);
+ return result;
+ }
+
+ const auto pathId = path.Base()->PathId;
+ result->SetPathId(pathId.LocalPathId);
+
+ auto guard = context.DbGuard();
+ context.MemChanges.GrabNewTxState(context.SS, OperationId);
+ context.MemChanges.GrabPath(context.SS, pathId);
+ context.MemChanges.GrabPath(context.SS, path->ParentPathId);
+ context.MemChanges.GrabExternalTable(context.SS, pathId);
+
+ context.DbChanges.PersistTxState(OperationId);
+ context.DbChanges.PersistPath(pathId);
+ context.DbChanges.PersistPath(path->ParentPathId);
+
+ Y_VERIFY(!context.SS->FindTx(OperationId));
+ TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxDropExternalTable, path.Base()->PathId);
+ txState.State = TTxState::Propose;
+ txState.MinStep = TStepId(1);
+
+ path.Base()->PathState = TPathElement::EPathState::EPathStateDrop;
+ path.Base()->DropTxId = OperationId.GetTxId();
+ path.Base()->LastTxId = OperationId.GetTxId();
+
+ IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, path, context.SS, context.OnComplete);
+
+ context.OnComplete.ActivateTx(OperationId);
+
+ SetState(NextState());
+ return result;
+ }
+
+ void AbortPropose(TOperationContext& context) override {
+ LOG_N("TDropExternalTable AbortPropose"
+ << ": opId# " << OperationId);
+ }
+
+ void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override {
+ LOG_N("TDropExternalTable AbortUnsafe"
+ << ": opId# " << OperationId
+ << ", txId# " << forceDropTxId);
+ context.OnComplete.DoneOperation(OperationId);
+ }
+};
+
+}
+
+namespace NKikimr::NSchemeShard {
+
+ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, const TTxTransaction& tx) {
+ return MakeSubOperation<TDropExternalTable>(id, tx);
+}
+
+ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, TTxState::ETxState state) {
+ Y_VERIFY(state != TTxState::Invalid);
+ return MakeSubOperation<TDropExternalTable>(id, state);
+}
+
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp
index 893ecaeefa..da529e7c35 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp
@@ -82,6 +82,14 @@ void TMemoryChanges::GrabLongLock(TSchemeShard* ss, const TPathId& pathId, TTxId
LockedPaths.emplace(pathId, lockTxId); // will be restored on UnDo()
}
+void TMemoryChanges::GrabNewExternalTable(TSchemeShard* ss, const TPathId& pathId) {
+ GrabNew(pathId, ss->ExternalTables, ExternalTables);
+}
+
+void TMemoryChanges::GrabExternalTable(TSchemeShard* ss, const TPathId& pathId) {
+ Grab<TExternalTableInfo>(pathId, ss->ExternalTables, ExternalTables);
+}
+
void TMemoryChanges::UnDo(TSchemeShard* ss) {
// be aware of the order of grab & undo ops
// stack is the best way to manage it right
@@ -181,6 +189,16 @@ void TMemoryChanges::UnDo(TSchemeShard* ss) {
}
TxStates.pop();
}
+
+ while (ExternalTables) {
+ const auto& [id, elem] = ExternalTables.top();
+ if (elem) {
+ ss->ExternalTables[id] = elem;
+ } else {
+ ss->ExternalTables.erase(id);
+ }
+ ExternalTables.pop();
+ }
}
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h
index f5b73942c4..6c9032b12b 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h
@@ -39,6 +39,9 @@ class TMemoryChanges: public TSimpleRefCount<TMemoryChanges> {
using TTxState = std::pair<TOperationId, THolder<TTxState>>;
TStack<TTxState> TxStates;
+ using TExternalTableState = std::pair<TPathId, TExternalTableInfo::TPtr>;
+ TStack<TExternalTableState> ExternalTables;
+
public:
~TMemoryChanges() = default;
@@ -66,6 +69,9 @@ public:
void GrabNewLongLock(TSchemeShard* ss, const TPathId& pathId);
void GrabLongLock(TSchemeShard* ss, const TPathId& pathId, TTxId lockTxId);
+ void GrabNewExternalTable(TSchemeShard* ss, const TPathId& pathId);
+ void GrabExternalTable(TSchemeShard* ss, const TPathId& pathId);
+
void UnDo(TSchemeShard* ss);
};
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h
index 41526b52f5..b6e0d7eba4 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h
@@ -369,6 +369,14 @@ ISubOperationBase::TPtr CreateDropTableIndexAtMainTable(TOperationId id, TTxStat
ISubOperationBase::TPtr CreateUpdateMainTableOnIndexMove(TOperationId id, const TTxTransaction& tx);
ISubOperationBase::TPtr CreateUpdateMainTableOnIndexMove(TOperationId id, TTxState::ETxState state);
+// External Table
+// Create
+ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, const TTxTransaction& tx);
+ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, TTxState::ETxState state);
+// Drop
+ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, const TTxTransaction& tx);
+ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, TTxState::ETxState state);
+
/// CDC
// Create
TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp
index 7bb8f2fe8f..9f67b76510 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp
@@ -303,6 +303,7 @@ public:
auto migrateShards = event->Record.MutableShards();
switch (path.Base()->PathType) {
case NKikimrSchemeOp::EPathType::EPathTypeDir:
+ case NKikimrSchemeOp::EPathType::EPathTypeExternalTable:
Y_VERIFY(!path.Base()->IsRoot());
//no shards
break;
diff --git a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp
index 9b4d522f15..a65626eef2 100644
--- a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp
@@ -186,6 +186,12 @@ TString DefineUserOperationName(NKikimrSchemeOp::EOperationType type) {
return "ALTER BLOB DEPOT";
case NKikimrSchemeOp::EOperationType::ESchemeOpDropBlobDepot:
return "DROP BLOB DEPOT";
+ case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable:
+ return "CREATE EXTERNAL TABLE";
+ case NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable:
+ return "DROP EXTERNAL TABLE";
+ case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExternalTable:
+ return "ALTER EXTERNAL TABLE";
}
Y_FAIL("switch should cover all operation types");
}
@@ -445,6 +451,15 @@ TVector<TString> ExtractChangingPaths(const NKikimrSchemeOp::TModifyScheme& tx)
result.emplace_back(NKikimr::JoinPath({tx.GetMoveIndex().GetTablePath(), tx.GetMoveIndex().GetSrcPath()}));
result.emplace_back(NKikimr::JoinPath({tx.GetMoveIndex().GetTablePath(), tx.GetMoveIndex().GetDstPath()}));
break;
+ case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable:
+ result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetCreateExternalTable().GetName()}));
+ break;
+ case NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable:
+ result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetDrop().GetName()}));
+ break;
+ case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExternalTable:
+ // TODO: unimplemented
+ break;
}
return result;
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
index 5804e7c5b6..117351ed00 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -1373,6 +1373,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T
case TTxState::TxCreateSequence:
case TTxState::TxCreateReplication:
case TTxState::TxCreateBlobDepot:
+ case TTxState::TxCreateExternalTable:
return TPathElement::EPathState::EPathStateCreate;
case TTxState::TxAlterPQGroup:
case TTxState::TxAlterTable:
@@ -1403,6 +1404,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T
case TTxState::TxAlterReplication:
case TTxState::TxAlterBlobDepot:
case TTxState::TxUpdateMainTableOnIndexMove:
+ case TTxState::TxAlterExternalTable:
return TPathElement::EPathState::EPathStateAlter;
case TTxState::TxDropTable:
case TTxState::TxDropPQGroup:
@@ -1421,6 +1423,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T
case TTxState::TxDropSequence:
case TTxState::TxDropReplication:
case TTxState::TxDropBlobDepot:
+ case TTxState::TxDropExternalTable:
return TPathElement::EPathState::EPathStateDrop;
case TTxState::TxBackup:
return TPathElement::EPathState::EPathStateBackup;
@@ -2588,6 +2591,56 @@ void TSchemeShard::PersistRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId, const
}
}
+void TSchemeShard::PersistExternalTable(NIceDb::TNiceDb &db, TPathId pathId, const TExternalTableInfo::TPtr externalTableInfo) {
+ Y_VERIFY(IsLocalId(pathId));
+
+ db.Table<Schema::ExternalTable>().Key(pathId.OwnerId, pathId.LocalPathId).Update(
+ NIceDb::TUpdate<Schema::ExternalTable::SourceType>{externalTableInfo->SourceType},
+ NIceDb::TUpdate<Schema::ExternalTable::DataSourcePath>{externalTableInfo->DataSourcePath},
+ NIceDb::TUpdate<Schema::ExternalTable::Location>{externalTableInfo->Location},
+ NIceDb::TUpdate<Schema::ExternalTable::AlterVersion>{externalTableInfo->AlterVersion},
+ NIceDb::TUpdate<Schema::ExternalTable::Content>{externalTableInfo->Content});
+
+ for (auto col : externalTableInfo->Columns) {
+ ui32 colId = col.first;
+ const TTableInfo::TColumn& cinfo = col.second;
+ TString typeData;
+ auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType);
+ if (columnType.TypeInfo) {
+ Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData));
+ }
+ db.Table<Schema::MigratedColumns>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Update(
+ NIceDb::TUpdate<Schema::MigratedColumns::ColName>(cinfo.Name),
+ NIceDb::TUpdate<Schema::MigratedColumns::ColType>((ui32)columnType.TypeId),
+ NIceDb::TUpdate<Schema::MigratedColumns::ColTypeData>(typeData),
+ NIceDb::TUpdate<Schema::MigratedColumns::ColKeyOrder>(cinfo.KeyOrder),
+ NIceDb::TUpdate<Schema::MigratedColumns::CreateVersion>(cinfo.CreateVersion),
+ NIceDb::TUpdate<Schema::MigratedColumns::DeleteVersion>(cinfo.DeleteVersion),
+ NIceDb::TUpdate<Schema::MigratedColumns::Family>(cinfo.Family),
+ NIceDb::TUpdate<Schema::MigratedColumns::DefaultKind>(cinfo.DefaultKind),
+ NIceDb::TUpdate<Schema::MigratedColumns::DefaultValue>(cinfo.DefaultValue),
+ NIceDb::TUpdate<Schema::MigratedColumns::NotNull>(cinfo.NotNull));
+ }
+}
+
+void TSchemeShard::PersistRemoveExternalTable(NIceDb::TNiceDb& db, TPathId pathId)
+{
+ Y_VERIFY(IsLocalId(pathId));
+ if (ExternalTables.contains(pathId)) {
+ auto externalTableInfo = ExternalTables.at(pathId);
+
+ for (auto col : externalTableInfo->Columns) {
+ const ui32 colId = col.first;
+ db.Table<Schema::MigratedColumns>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Delete();
+ }
+
+ ExternalTables.erase(pathId);
+ DecrementPathDbRefCount(pathId);
+ }
+
+ db.Table<Schema::ExternalTable>().Key(pathId.OwnerId, pathId.LocalPathId).Delete();
+}
+
void TSchemeShard::PersistRemoveRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId) {
Y_VERIFY(IsLocalId(pathId));
@@ -3819,6 +3872,13 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co
Y_FAIL_S("BlobDepot for path " << pathId << " not found");
}
break;
+ case NKikimrSchemeOp::EPathType::EPathTypeExternalTable: {
+ auto it = ExternalTables.find(pathId);
+ Y_VERIFY(it != ExternalTables.end());
+ result.SetExternalTableVersion(it->second->AlterVersion);
+ generalVersion += result.GetExternalTableVersion();
+ break;
+ }
case NKikimrSchemeOp::EPathType::EPathTypeInvalid: {
Y_UNREACHABLE();
@@ -4567,6 +4627,9 @@ void TSchemeShard::UncountNode(TPathElement::TPtr node) {
case TPathElement::EPathType::EPathTypeBlobDepot:
TabletCounters->Simple()[COUNTER_BLOB_DEPOT_COUNT].Sub(1);
break;
+ case TPathElement::EPathType::EPathTypeExternalTable:
+ TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Sub(1);
+ break;
case TPathElement::EPathType::EPathTypeInvalid:
Y_FAIL("impossible path type");
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h
index b6b276f1ae..37c0803cd5 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.h
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.h
@@ -209,6 +209,7 @@ public:
THashMap<TPathId, TFileStoreInfo::TPtr> FileStoreInfos;
THashMap<TPathId, TKesusInfo::TPtr> KesusInfos;
THashMap<TPathId, TOlapStoreInfo::TPtr> OlapStores;
+ THashMap<TPathId, TExternalTableInfo::TPtr> ExternalTables;
TTablesStorage ColumnTables;
@@ -729,6 +730,10 @@ public:
void PersistStorageBillingTime(NIceDb::TNiceDb& db);
+ // ExternalTable
+ void PersistExternalTable(NIceDb::TNiceDb &db, TPathId pathId, const TExternalTableInfo::TPtr externalTable);
+ void PersistRemoveExternalTable(NIceDb::TNiceDb& db, TPathId pathId);
+
TTabletId GetGlobalHive(const TActorContext& ctx) const;
enum class EHiveSelection : uint8_t {
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
index 304b441cd1..b94a63e9a1 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -2897,6 +2897,17 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
NKikimrSchemeOp::TIndexBuildConfig SerializeToProto(TSchemeShard* ss) const;
};
+struct TExternalTableInfo: TSimpleRefCount<TExternalTableInfo> {
+ using TPtr = TIntrusivePtr<TExternalTableInfo>;
+
+ TString SourceType;
+ TString DataSourcePath;
+ TString Location;
+ ui64 AlterVersion = 0;
+ THashMap<ui32, TTableInfo::TColumn> Columns;
+ TString Content;
+};
+
bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl,
const THashMap<ui32, TTableInfo::TColumn>& sourceColumns,
const THashMap<ui32, TTableInfo::TColumn>& alterColumns,
diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp
index b94a2fb456..c68fbbec34 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp
@@ -742,6 +742,19 @@ const TPath::TChecker& TPath::TChecker::PQReservedStorageLimit(ui64 delta, EStat
return *this;
}
+const TPath::TChecker& TPath::TChecker::IsExternalTable(EStatus status) const {
+ if (Failed) {
+ return *this;
+ }
+
+ if (Path.Base()->IsExternalTable()) {
+ return *this;
+ }
+
+ return Fail(status, TStringBuilder() << "path is not a external table"
+ << " (" << BasicPathInfo(Path.Base()) << ")");
+}
+
const TPath::TChecker& TPath::TChecker::PathShardsLimit(ui64 delta, EStatus status) const {
if (Failed) {
return *this;
diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h
index fa328fe1bd..172dbcc3af 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path.h
+++ b/ydb/core/tx/schemeshard/schemeshard_path.h
@@ -85,6 +85,7 @@ public:
const TChecker& IsValidACL(const TString& acl, EStatus status = EStatus::StatusInvalidParameter) const;
const TChecker& PQPartitionsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const;
const TChecker& PQReservedStorageLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const;
+ const TChecker& IsExternalTable(EStatus status = EStatus::StatusNameConflict) const;
};
public:
diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
index bb6c2e20c5..b5e8b82a80 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
@@ -777,6 +777,40 @@ void TPathDescriber::DescribeBlobDepot(const TPath& path) {
Self->DescribeBlobDepot(path->PathId, path->Name, *Result->Record.MutablePathDescription()->MutableBlobDepotDescription());
}
+void TPathDescriber::DescribeExternalTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl) {
+ const NScheme::TTypeRegistry* typeRegistry = AppData(ctx)->TypeRegistry;
+ auto it = Self->ExternalTables.FindPtr(pathId);
+ Y_VERIFY(it, "ExternalTable is not found");
+ TExternalTableInfo::TPtr externalTableInfo = *it;
+
+ auto entry = Result->Record.MutablePathDescription()->MutableExternalTableDescription();
+ entry->SetName(pathEl->Name);
+ PathIdFromPathId(pathId, entry->MutablePathId());
+ entry->SetSourceType(externalTableInfo->SourceType);
+ entry->SetDataSourcePath(externalTableInfo->DataSourcePath);
+ entry->SetLocation(externalTableInfo->Location);
+ entry->SetVersion(externalTableInfo->AlterVersion);
+
+ entry->MutableColumns()->Reserve(externalTableInfo->Columns.size());
+ for (auto col : externalTableInfo->Columns) {
+ const auto& cinfo = col.second;
+ if (cinfo.IsDropped())
+ continue;
+
+ auto colDescr = entry->AddColumns();
+ colDescr->SetName(cinfo.Name);
+ colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType.GetTypeId())); // TODO: no pg type details in string type
+ auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType);
+ colDescr->SetTypeId(columnType.TypeId);
+ if (columnType.TypeInfo) {
+ *colDescr->MutableTypeInfo() = *columnType.TypeInfo;
+ }
+ colDescr->SetId(cinfo.Id);
+ colDescr->SetNotNull(cinfo.NotNull);
+ }
+ entry->SetContent(externalTableInfo->Content);
+}
+
THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe(const TActorContext& ctx) {
TPathId pathId = Params.HasPathId() ? TPathId(Params.GetSchemeshardId(), Params.GetPathId()) : InvalidPathId;
TString pathStr = Params.GetPath();
@@ -904,6 +938,9 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe
case NKikimrSchemeOp::EPathTypeBlobDepot:
DescribeBlobDepot(path);
break;
+ case NKikimrSchemeOp::EPathTypeExternalTable:
+ DescribeExternalTable(ctx, base->PathId, base);
+ break;
case NKikimrSchemeOp::EPathTypeInvalid:
Y_UNREACHABLE();
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.h b/ydb/core/tx/schemeshard/schemeshard_path_describer.h
index f0d1902519..05f44ceb3f 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path_describer.h
+++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.h
@@ -43,6 +43,7 @@ class TPathDescriber {
void DescribeSequence(TPathId pathId, TPathElement::TPtr pathEl);
void DescribeReplication(TPathId pathId, TPathElement::TPtr pathEl);
void DescribeBlobDepot(const TPath& path);
+ void DescribeExternalTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl);
public:
explicit TPathDescriber(TSchemeShard* self, NKikimrSchemeOp::TDescribePath&& params)
diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp
index 3deab09737..91d14a7a13 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp
@@ -164,6 +164,10 @@ TVirtualTimestamp TPathElement::GetDropTS() const {
return TVirtualTimestamp(StepDropped, DropTxId);
}
+bool TPathElement::IsExternalTable() const {
+ return PathType == EPathType::EPathTypeExternalTable;
+}
+
void TPathElement::SetDropped(TStepId step, TTxId txId) {
PathState = EPathState::EPathStateNotExist;
StepDropped = step;
diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h
index 0ad50bd3dd..25edaff694 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path_element.h
+++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h
@@ -378,6 +378,7 @@ public:
bool IsLikeDirectory() const;
bool HasActiveChanges() const;
bool IsCreateFinished() const;
+ bool IsExternalTable() const;
TVirtualTimestamp GetCreateTS() const;
TVirtualTimestamp GetDropTS() const;
void SetDropped(TStepId step, TTxId txId);
diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h
index 007959c486..509f6a1418 100644
--- a/ydb/core/tx/schemeshard/schemeshard_schema.h
+++ b/ydb/core/tx/schemeshard/schemeshard_schema.h
@@ -1619,6 +1619,19 @@ struct Schema : NIceDb::Schema {
using TColumns = TableColumns<PathId, AlterVersion, Description>;
};
+ struct ExternalTable : Table<104> {
+ struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; };
+ struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; };
+ struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {};
+ struct SourceType : Column<4, NScheme::NTypeIds::Utf8> {};
+ struct DataSourcePath : Column<5, NScheme::NTypeIds::Utf8> {};
+ struct Location : Column<6, NScheme::NTypeIds::Utf8> {};
+ struct Content : Column<7, NScheme::NTypeIds::String> {};
+
+ using TKey = TableKey<OwnerPathId, LocalPathId>;
+ using TColumns = TableColumns<OwnerPathId, LocalPathId, SourceType, DataSourcePath, Location, AlterVersion, Content>;
+ };
+
using TTables = SchemaTables<
Paths,
TxInFlight,
@@ -1721,7 +1734,8 @@ struct Schema : NIceDb::Schema {
Replications,
ReplicationsAlterData,
BlobDepots,
- CdcStreamScanShardStatus
+ CdcStreamScanShardStatus,
+ ExternalTable
>;
static constexpr ui64 SysParam_NextPathId = 1;
diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h
index c150e03b85..55055400a6 100644
--- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h
+++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h
@@ -120,6 +120,9 @@ struct TTxState {
item(TxAlterExtSubDomainCreateHive, 74) \
item(TxAlterCdcStreamAtTableDropSnapshot, 75) \
item(TxDropCdcStreamAtTableDropSnapshot, 76) \
+ item(TxCreateExternalTable, 77) \
+ item(TxDropExternalTable, 78) \
+ item(TxAlterExternalTable, 79) \
// TX_STATE_TYPE_ENUM
@@ -327,6 +330,7 @@ struct TTxState {
case TxCreateSequence:
case TxCreateReplication:
case TxCreateBlobDepot:
+ case TxCreateExternalTable:
return true;
case TxInitializeBuildIndex: //this is more like alter
case TxCreateCdcStreamAtTable:
@@ -357,6 +361,7 @@ struct TTxState {
case TxDropReplication:
case TxDropBlobDepot:
case TxUpdateMainTableOnIndexMove:
+ case TxDropExternalTable:
return false;
case TxAlterPQGroup:
case TxAlterTable:
@@ -385,6 +390,7 @@ struct TTxState {
case TxAlterSequence:
case TxAlterReplication:
case TxAlterBlobDepot:
+ case TxAlterExternalTable:
return false;
case TxMoveTable:
case TxMoveTableIndex:
@@ -414,6 +420,7 @@ struct TTxState {
case TxDropSequence:
case TxDropReplication:
case TxDropBlobDepot:
+ case TxDropExternalTable:
return true;
case TxMkDir:
case TxCreateTable:
@@ -445,6 +452,7 @@ struct TTxState {
case TxDropCdcStreamAtTable:
case TxDropCdcStreamAtTableDropSnapshot:
case TxUpdateMainTableOnIndexMove:
+ case TxCreateExternalTable:
return false;
case TxAlterPQGroup:
case TxAlterTable:
@@ -473,6 +481,7 @@ struct TTxState {
case TxAlterSequence:
case TxAlterReplication:
case TxAlterBlobDepot:
+ case TxAlterExternalTable:
return false;
case TxMoveTable:
case TxMoveTableIndex:
@@ -506,6 +515,7 @@ struct TTxState {
case TxDropTableIndex:
case TxRmDir:
case TxFinalizeBuildIndex:
+ case TxDropExternalTable:
return false;
case TxMkDir:
case TxCreateTable:
@@ -535,6 +545,7 @@ struct TTxState {
case TxDropCdcStreamAtTable:
case TxDropCdcStreamAtTableDropSnapshot:
case TxUpdateMainTableOnIndexMove:
+ case TxCreateExternalTable:
return false;
case TxAlterPQGroup:
case TxAlterTable:
@@ -564,6 +575,7 @@ struct TTxState {
case TxAlterSequence:
case TxAlterReplication:
case TxAlterBlobDepot:
+ case TxAlterExternalTable:
return false;
case TxInvalid:
Y_VERIFY_DEBUG("UNREACHABLE");
@@ -657,6 +669,8 @@ struct TTxState {
case NKikimrSchemeOp::ESchemeOpMoveIndex: return TxInvalid;
case NKikimrSchemeOp::ESchemeOpAllocatePersQueueGroup: return TxAllocatePQ;
case NKikimrSchemeOp::ESchemeOpDeallocatePersQueueGroup: return TxInvalid;
+ case NKikimrSchemeOp::ESchemeOpCreateExternalTable: return TxCreateExternalTable;
+ case NKikimrSchemeOp::ESchemeOpAlterExternalTable: return TxAlterExternalTable;
default: return TxInvalid;
}
}
diff --git a/ydb/core/tx/schemeshard/ut_external_table.cpp b/ydb/core/tx/schemeshard/ut_external_table.cpp
new file mode 100644
index 0000000000..e9eb58d7c4
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table.cpp
@@ -0,0 +1,237 @@
+#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
+
+using namespace NKikimr::NSchemeShard;
+using namespace NKikimr;
+using namespace NKikimrSchemeOp;
+using namespace NSchemeShardUT_Private;
+
+Y_UNIT_TEST_SUITE(TExternalTableTest) {
+ Y_UNIT_TEST(CreateExternalTable) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ TestCreateExternalTable(runtime, txId++, "/MyRoot",
+ R"(Name: "external_table1")",
+ {NKikimrScheme::StatusAccepted});
+
+ env.TestWaitNotification(runtime, 100);
+
+ TestLs(runtime, "/MyRoot/external_table1", false, NLs::PathExist);
+ }
+
+ Y_UNIT_TEST(DropExternalTable) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ TestCreateExternalTable(runtime, txId++, "/MyRoot",
+ R"(Name: "external_table1")",
+ {NKikimrScheme::StatusAccepted});
+
+ env.TestWaitNotification(runtime, 100);
+
+ TestLs(runtime, "/MyRoot/external_table1", false, NLs::PathExist);
+
+ TestDropExternalTable(runtime, ++txId, "/MyRoot", "external_table1");
+ env.TestWaitNotification(runtime, txId);
+
+ TestLs(runtime, "/MyRoot/external_table1", false, NLs::PathNotExist);
+ }
+
+ using TRuntimeTxFn = std::function<void(TTestBasicRuntime&, ui64)>;
+
+ void DropTwice(const TString& path, TRuntimeTxFn createFn, TRuntimeTxFn dropFn) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ createFn(runtime, ++txId);
+ env.TestWaitNotification(runtime, txId);
+
+ dropFn(runtime, ++txId);
+ dropFn(runtime, ++txId);
+ TestModificationResult(runtime, txId - 1);
+
+ auto ev = runtime.GrabEdgeEvent<TEvSchemeShard::TEvModifySchemeTransactionResult>();
+ UNIT_ASSERT(ev);
+
+ const auto& record = ev->Record;
+ UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId);
+ UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusMultipleModifications);
+ UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 1);
+
+ env.TestWaitNotification(runtime, txId - 1);
+ TestDescribeResult(DescribePath(runtime, path), {
+ NLs::PathNotExist
+ });
+ }
+
+ Y_UNIT_TEST(DropTableTwice) {
+ auto createFn = [](TTestBasicRuntime& runtime, ui64 txId) {
+ TestCreateExternalTable(runtime, txId, "/MyRoot", R"(
+ Name: "ExternalTable"
+ Columns { Name: "key" Type: "Uint64" }
+ Columns { Name: "value" Type: "Utf8" }
+ )");
+ };
+
+ auto dropFn = [](TTestBasicRuntime& runtime, ui64 txId) {
+ AsyncDropExternalTable(runtime, txId, "/MyRoot", "ExternalTable");
+ };
+
+ DropTwice("/MyRoot/ExternalTable", createFn, dropFn);
+ }
+
+ Y_UNIT_TEST(ParallelCreateExternalTable) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 123;
+
+ AsyncMkDir(runtime, ++txId, "/MyRoot", "DirA");
+ AsyncCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "ExternalTable1"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ AsyncCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "ExternalTable2"
+ Columns { Name: "key1" Type: "Uint32"}
+ Columns { Name: "key2" Type: "Utf8"}
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ TestModificationResult(runtime, txId-2, NKikimrScheme::StatusAccepted);
+ TestModificationResult(runtime, txId-1, NKikimrScheme::StatusAccepted);
+ TestModificationResult(runtime, txId, NKikimrScheme::StatusAccepted);
+
+ env.TestWaitNotification(runtime, {txId, txId-1, txId-2});
+
+ TestDescribe(runtime, "/MyRoot/DirA/ExternalTable1");
+ TestDescribe(runtime, "/MyRoot/DirA/ExternalTable2");
+
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"),
+ {NLs::PathVersionEqual(7)});
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/ExternalTable1"),
+ {NLs::PathVersionEqual(2)});
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/ExternalTable2"),
+ {NLs::PathVersionEqual(2)});
+ }
+
+ Y_UNIT_TEST(ParallelCreateSameExternalTable) {
+ using ESts = NKikimrScheme::EStatus;
+
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 123;
+
+ TString tableConfig = R"(Name: "NilNoviSubLuna"
+ Columns { Name: "key" Type: "Uint64"}
+ Columns { Name: "value" Type: "Uint64"})";
+
+ AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig);
+ AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig);
+ AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig);
+
+ ui64 sts[3];
+ sts[0] = TestModificationResults(runtime, txId-2, {ESts::StatusAccepted, ESts::StatusMultipleModifications, ESts::StatusAlreadyExists});
+ sts[1] = TestModificationResults(runtime, txId-1, {ESts::StatusAccepted, ESts::StatusMultipleModifications, ESts::StatusAlreadyExists});
+ sts[2] = TestModificationResults(runtime, txId, {ESts::StatusAccepted, ESts::StatusMultipleModifications, ESts::StatusAlreadyExists});
+
+ for (ui32 i=0; i<3; ++i) {
+ if (sts[i] == ESts::StatusAlreadyExists) {
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/NilNoviSubLuna"),
+ {NLs::Finished,
+ NLs::IsExternalTable});
+ }
+
+ if (sts[i] == ESts::StatusMultipleModifications) {
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/NilNoviSubLuna"),
+ {NLs::Finished,
+ NLs::IsExternalTable});
+ }
+ }
+
+ env.TestWaitNotification(runtime, {txId-2, txId-1, txId});
+
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/NilNoviSubLuna"),
+ {NLs::Finished,
+ NLs::IsExternalTable,
+ NLs::PathVersionEqual(2)});
+
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig, {ESts::StatusAlreadyExists});
+
+ }
+
+
+ Y_UNIT_TEST(ReadOnlyMode) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 123;
+
+ AsyncMkDir(runtime, ++txId, "/MyRoot", "SubDirA");
+ AsyncCreateExternalTable(runtime, ++txId, "/MyRoot",
+ R"(Name: "ExternalTable1"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ // Set ReadOnly
+ SetSchemeshardReadOnlyMode(runtime, true);
+ TActorId sender = runtime.AllocateEdgeActor();
+ RebootTablet(runtime, TTestTxConfig::SchemeShard, sender);
+
+ // Verify that table creation successfully finished
+ env.TestWaitNotification(runtime, txId);
+
+ // Check that describe works
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/SubDirA"),
+ {NLs::Finished});
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable1"),
+ {NLs::Finished,
+ NLs::IsExternalTable});
+
+ // Check that new modifications fail
+ TestMkDir(runtime, ++txId, "/MyRoot", "SubDirBBBB", {NKikimrScheme::StatusReadOnly});
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot",
+ R"(Name: "ExternalTable1"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})",
+ {NKikimrScheme::StatusReadOnly});
+
+ // Disable ReadOnly
+ SetSchemeshardReadOnlyMode(runtime, false);
+ sender = runtime.AllocateEdgeActor();
+ RebootTablet(runtime, TTestTxConfig::SchemeShard, sender);
+
+ // Check that modifications now work again
+ TestMkDir(runtime, ++txId, "/MyRoot", "SubDirBBBB");
+ }
+
+ Y_UNIT_TEST(SchemeErrors) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 123;
+
+ TestMkDir(runtime, ++txId, "/MyRoot", "DirA");
+ env.TestWaitNotification(runtime, txId);
+
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "Table2"
+ Columns { Name: "RowId" Type: "BlaBlaType"})",
+ {NKikimrScheme::StatusSchemeError});
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "Table2"
+ Columns { Name: "" Type: "Uint64"})",
+ {NKikimrScheme::StatusSchemeError});
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "Table2"
+ Columns { Name: "RowId" TypeId: 27})",
+ {NKikimrScheme::StatusSchemeError});
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "Table2"
+ Columns { Name: "RowId" })",
+ {NKikimrScheme::StatusSchemeError});
+ TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA",
+ R"(Name: "Table2"
+ Columns { Name: "RowId" Type: "Uint64" Id: 2}
+ Columns { Name: "RowId2" Type: "Uint64" Id: 2 })",
+ {NKikimrScheme::StatusSchemeError});
+ }
+}
diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt
new file mode 100644
index 0000000000..3b8e4769f6
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt
@@ -0,0 +1,82 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tx-schemeshard-ut_external_table)
+target_compile_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard
+)
+target_link_libraries(ydb-core-tx-schemeshard-ut_external_table PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-tx-schemeshard
+ library-cpp-getopt
+ cpp-regex-pcre
+ library-cpp-svnversion
+ core-testlib-default
+ ydb-core-tx
+ tx-schemeshard-ut_helpers
+ udf-service-exception_policy
+)
+target_link_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ -Wl,-no_deduplicate
+ -Wl,-sdk_version,10.15
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-core-tx-schemeshard-ut_external_table
+ TEST_TARGET
+ ydb-core-tx-schemeshard-ut_external_table
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tx-schemeshard-ut_external_table)
diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..5b048e61f6
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,85 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tx-schemeshard-ut_external_table)
+target_compile_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard
+)
+target_link_libraries(ydb-core-tx-schemeshard-ut_external_table PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-lfalloc
+ cpp-testing-unittest_main
+ core-tx-schemeshard
+ library-cpp-getopt
+ cpp-regex-pcre
+ library-cpp-svnversion
+ core-testlib-default
+ ydb-core-tx
+ tx-schemeshard-ut_helpers
+ udf-service-exception_policy
+)
+target_link_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-core-tx-schemeshard-ut_external_table
+ TEST_TARGET
+ ydb-core-tx-schemeshard-ut_external_table
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tx-schemeshard-ut_external_table)
diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt
new file mode 100644
index 0000000000..f1418da3f2
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt
@@ -0,0 +1,87 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tx-schemeshard-ut_external_table)
+target_compile_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard
+)
+target_link_libraries(ydb-core-tx-schemeshard-ut_external_table PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-tx-schemeshard
+ library-cpp-getopt
+ cpp-regex-pcre
+ library-cpp-svnversion
+ core-testlib-default
+ ydb-core-tx
+ tx-schemeshard-ut_helpers
+ udf-service-exception_policy
+)
+target_link_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-tx-schemeshard-ut_external_table PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-core-tx-schemeshard-ut_external_table
+ TEST_TARGET
+ ydb-core-tx-schemeshard-ut_external_table
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tx-schemeshard-ut_external_table)
diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt
new file mode 100644
index 0000000000..5bb4faffb4
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt
@@ -0,0 +1,15 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin.txt)
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux.txt)
+endif()
diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp b/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp
new file mode 100644
index 0000000000..ea276bf960
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp
@@ -0,0 +1,214 @@
+#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
+
+#include <ydb/core/tx/datashard/datashard.h>
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+
+#include <google/protobuf/text_format.h>
+
+using namespace NKikimr;
+using namespace NSchemeShard;
+using namespace NSchemeShardUT_Private;
+
+Y_UNIT_TEST_SUITE(TExternalTableTestReboots) {
+ Y_UNIT_TEST(CreateExternalTableWithReboots) {
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ AsyncMkDir(runtime, ++t.TxId, "/MyRoot", "DirExternalTable");
+
+ AsyncCreateExternalTable(runtime, ++t.TxId, "/MyRoot/DirExternalTable", R"(
+ Name: "external_table1"
+ DataSourcePath: "/MySource"
+ Columns { Name: "a" Type: "Int32" NotNull: true }
+ Columns { Name: "b" Type: "Int32" NotNull: true }
+ )");
+
+ t.TestEnv->TestWaitNotification(runtime, {t.TxId, t.TxId-1});
+
+ {
+ TInactiveZone inactive(activeZone);
+ auto describeResult = DescribePath(runtime, "/MyRoot/DirExternalTable/external_table1");
+ TestDescribeResult(describeResult, {NLs::Finished});
+
+ UNIT_ASSERT(describeResult.GetPathDescription().HasExternalTableDescription());
+ const auto& externalTableDescription = describeResult.GetPathDescription().GetExternalTableDescription();
+ UNIT_ASSERT_VALUES_EQUAL(externalTableDescription.GetName(), "external_table1");
+ UNIT_ASSERT_VALUES_EQUAL(externalTableDescription.GetDataSourcePath(), "/MySource");
+ UNIT_ASSERT_VALUES_EQUAL(externalTableDescription.GetVersion(), 1);
+ auto& columns = externalTableDescription.GetColumns();
+ UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(columns.Get(0).GetName(), "a");
+ UNIT_ASSERT_VALUES_EQUAL(columns.Get(0).GetType(), "Int32");
+ UNIT_ASSERT_VALUES_EQUAL(columns.Get(0).GetNotNull(), true);
+ UNIT_ASSERT_VALUES_EQUAL(columns.Get(1).GetName(), "b");
+ UNIT_ASSERT_VALUES_EQUAL(columns.Get(1).GetType(), "Int32");
+ UNIT_ASSERT_VALUES_EQUAL(columns.Get(1).GetNotNull(), true);
+ }
+ });
+ }
+
+ Y_UNIT_TEST(ParallelCreateDrop) { //+
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ AsyncCreateExternalTable(runtime, ++t.TxId, "/MyRoot", R"(
+ Name: "DropMe"
+ Columns { Name: "RowId" Type: "Uint64" }
+ Columns { Name: "Value" Type: "Utf8" }
+ )");
+ AsyncDropExternalTable(runtime, ++t.TxId, "/MyRoot", "DropMe");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId-1);
+
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "DropMe");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ {
+ TInactiveZone inactive(activeZone);
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/DropMe"),
+ {NLs::PathNotExist});
+ }
+ });
+ }
+
+ Y_UNIT_TEST(SimpleDropExternalTableWithReboots) {
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ {
+ TInactiveZone inactive(activeZone);
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ "Name: \"ExternalTable\""
+ "Columns { Name: \"RowId\" Type: \"Uint64\"}"
+ "Columns { Name: \"Value\" Type: \"Utf8\"}");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+ }
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ {
+ TInactiveZone inactive(activeZone);
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"),
+ {NLs::PathNotExist});
+ }
+ });
+ }
+
+ Y_UNIT_TEST(SimpleDropExternalTableWithReboots2) { //+
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ {
+ TInactiveZone inactive(activeZone);
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ "Name: \"ExternalTable\""
+ "Columns { Name: \"RowId\" Type: \"Uint64\"}"
+ "Columns { Name: \"Value\" Type: \"Utf8\"}");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+ }
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ {
+ TInactiveZone inactive(activeZone);
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"),
+ {NLs::PathNotExist});
+ }
+ });
+ }
+
+ Y_UNIT_TEST(DropExternalTableWithReboots) {
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ {
+ TInactiveZone inactive(activeZone);
+ TestCreateExternalTable(runtime, t.TxId, "/MyRoot",
+ R"(Name: "ExternalTable"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+ }
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ {
+ TInactiveZone inactive(activeZone);
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"),
+ {NLs::PathNotExist});
+
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ R"(Name: "ExternalTable"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"),
+ {NLs::PathNotExist});
+ }
+ });
+ }
+
+ Y_UNIT_TEST(CreateDroppedExternalTableWithReboots) {
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ {
+ TInactiveZone inactive(activeZone);
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ R"(Name: "ExternalTable"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+ }
+
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ R"(Name: "ExternalTable"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ {
+ TInactiveZone inactive(activeZone);
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+ }
+ });
+ }
+
+ Y_UNIT_TEST(CreateDroppedExternalTableAndDropWithReboots) { //+
+ TTestWithReboots t;
+ t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
+ {
+ TInactiveZone inactive(activeZone);
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ R"(Name: "ExternalTable"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot",
+ R"(Name: "ExternalTable"
+ Columns { Name: "RowId" Type: "Uint64"}
+ Columns { Name: "Value" Type: "Utf8"})");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+ }
+
+ TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable");
+ t.TestEnv->TestWaitNotification(runtime, t.TxId);
+
+ {
+ TInactiveZone inactive(activeZone);
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"),
+ {NLs::PathNotExist});
+ }
+ });
+ }
+}
diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt
new file mode 100644
index 0000000000..3af12236ae
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt
@@ -0,0 +1,82 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tx-schemeshard-ut_external_table_reboots)
+target_compile_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard
+)
+target_link_libraries(ydb-core-tx-schemeshard-ut_external_table_reboots PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-tx-schemeshard
+ library-cpp-getopt
+ cpp-regex-pcre
+ library-cpp-svnversion
+ core-testlib-default
+ ydb-core-tx
+ tx-schemeshard-ut_helpers
+ udf-service-exception_policy
+)
+target_link_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ -Wl,-no_deduplicate
+ -Wl,-sdk_version,10.15
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ TEST_TARGET
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tx-schemeshard-ut_external_table_reboots)
diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..e33bdbef26
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,85 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tx-schemeshard-ut_external_table_reboots)
+target_compile_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard
+)
+target_link_libraries(ydb-core-tx-schemeshard-ut_external_table_reboots PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-lfalloc
+ cpp-testing-unittest_main
+ core-tx-schemeshard
+ library-cpp-getopt
+ cpp-regex-pcre
+ library-cpp-svnversion
+ core-testlib-default
+ ydb-core-tx
+ tx-schemeshard-ut_helpers
+ udf-service-exception_policy
+)
+target_link_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ TEST_TARGET
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tx-schemeshard-ut_external_table_reboots)
diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt
new file mode 100644
index 0000000000..50b7782671
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt
@@ -0,0 +1,87 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tx-schemeshard-ut_external_table_reboots)
+target_compile_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard
+)
+target_link_libraries(ydb-core-tx-schemeshard-ut_external_table_reboots PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-tx-schemeshard
+ library-cpp-getopt
+ cpp-regex-pcre
+ library-cpp-svnversion
+ core-testlib-default
+ ydb-core-tx
+ tx-schemeshard-ut_helpers
+ udf-service-exception_policy
+)
+target_link_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ TEST_TARGET
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tx-schemeshard-ut_external_table_reboots
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tx-schemeshard-ut_external_table_reboots)
diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt
new file mode 100644
index 0000000000..5bb4faffb4
--- /dev/null
+++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt
@@ -0,0 +1,15 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin.txt)
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux.txt)
+endif()
diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
index bcbc7ba173..4fa6ca245b 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
@@ -888,6 +888,11 @@ namespace NSchemeShardUT_Private {
GENERIC_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableDrop)
DROP_BY_PATH_ID_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume)
+ // external table
+ GENERIC_HELPERS(CreateExternalTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateExternalTable)
+ GENERIC_HELPERS(DropExternalTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable, &NKikimrSchemeOp::TModifyScheme::MutableDrop)
+ DROP_BY_PATH_ID_HELPERS(DropExternalTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable)
+
#undef DROP_BY_PATH_ID_HELPERS
#undef GENERIC_WITH_ATTRS_HELPERS
#undef GENERIC_HELPERS
diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h
index f7786d1538..49c2f892fb 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h
+++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h
@@ -175,6 +175,12 @@ namespace NSchemeShardUT_Private {
DROP_BY_PATH_ID_HELPERS(DropTable);
GENERIC_HELPERS(DropTableIndex);
+
+ // external table
+ GENERIC_HELPERS(CreateExternalTable);
+ GENERIC_HELPERS(DropExternalTable);
+ DROP_BY_PATH_ID_HELPERS(DropExternalTable);
+
// backup & restore
GENERIC_HELPERS(Backup);
GENERIC_HELPERS(BackupToYt);
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
index 975b9ae199..ebde6a638a 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
@@ -389,6 +389,13 @@ void IsTable(const NKikimrScheme::TEvDescribeSchemeResult& record) {
UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeTable);
}
+void IsExternalTable(const NKikimrScheme::TEvDescribeSchemeResult& record) {
+ UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess);
+ const auto& pathDescr = record.GetPathDescription();
+ const auto& selfPath = pathDescr.GetSelf();
+ UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeExternalTable);
+}
+
TCheckFunc CheckColumns(const TString& name, const TSet<TString>& columns, const TSet<TString>& droppedColumns, const TSet<TString> keyColumns,
NKikimrSchemeOp::EPathState pathState) {
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
index 8043bd5fa1..0a4a266a1e 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
@@ -81,6 +81,7 @@ namespace NLs {
void IsTable(const NKikimrScheme::TEvDescribeSchemeResult& record);
+ void IsExternalTable(const NKikimrScheme::TEvDescribeSchemeResult& record);
TCheckFunc CheckColumns(const TString& name, const TSet<TString>& columns, const TSet<TString>& droppedColumns, const TSet<TString> keyColumns,
NKikimrSchemeOp::EPathState pathState = NKikimrSchemeOp::EPathState::EPathStateNoChanges);
void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult& record);
diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp
index ccd7e4ae46..ff5c90d2f5 100644
--- a/ydb/core/tx/tx_proxy/schemereq.cpp
+++ b/ydb/core/tx/tx_proxy/schemereq.cpp
@@ -150,6 +150,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> {
case NKikimrSchemeOp::ESchemeOpDropSequence:
case NKikimrSchemeOp::ESchemeOpDropReplication:
case NKikimrSchemeOp::ESchemeOpDropBlobDepot:
+ case NKikimrSchemeOp::ESchemeOpDropExternalTable:
return *modifyScheme.MutableDrop()->MutableName();
case NKikimrSchemeOp::ESchemeOpAlterTable:
@@ -324,6 +325,12 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> {
case NKikimrSchemeOp::ESchemeOpCreateBlobDepot:
case NKikimrSchemeOp::ESchemeOpAlterBlobDepot:
return *modifyScheme.MutableBlobDepot()->MutableName();
+
+ case NKikimrSchemeOp::ESchemeOpCreateExternalTable:
+ return *modifyScheme.MutableCreateExternalTable()->MutableName();
+
+ case NKikimrSchemeOp::ESchemeOpAlterExternalTable:
+ Y_FAIL("no implementation for ESchemeOpAlterExternalTable");
}
}
@@ -343,6 +350,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> {
case NKikimrSchemeOp::ESchemeOpCreateRtmrVolume:
case NKikimrSchemeOp::ESchemeOpCreateColumnStore:
case NKikimrSchemeOp::ESchemeOpCreateColumnTable:
+ case NKikimrSchemeOp::ESchemeOpCreateExternalTable:
return true;
default:
return false;
@@ -564,6 +572,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> {
case NKikimrSchemeOp::ESchemeOpAlterSequence:
case NKikimrSchemeOp::ESchemeOpAlterReplication:
case NKikimrSchemeOp::ESchemeOpAlterBlobDepot:
+ case NKikimrSchemeOp::ESchemeOpAlterExternalTable:
{
auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType());
toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme)));
@@ -583,7 +592,8 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> {
case NKikimrSchemeOp::ESchemeOpDropColumnTable:
case NKikimrSchemeOp::ESchemeOpDropSequence:
case NKikimrSchemeOp::ESchemeOpDropReplication:
- case NKikimrSchemeOp::ESchemeOpDropBlobDepot: {
+ case NKikimrSchemeOp::ESchemeOpDropBlobDepot:
+ case NKikimrSchemeOp::ESchemeOpDropExternalTable: {
auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType());
toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme)));
toResolve.RequiredAccess = NACLib::EAccessRights::RemoveSchema;
@@ -641,6 +651,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> {
case NKikimrSchemeOp::ESchemeOpCreateSequence:
case NKikimrSchemeOp::ESchemeOpCreateReplication:
case NKikimrSchemeOp::ESchemeOpCreateBlobDepot:
+ case NKikimrSchemeOp::ESchemeOpCreateExternalTable:
{
auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType());
toResolve.Path = workingDir;
diff --git a/ydb/core/viewer/browse.h b/ydb/core/viewer/browse.h
index 0ff4f33d94..930c5e971f 100644
--- a/ydb/core/viewer/browse.h
+++ b/ydb/core/viewer/browse.h
@@ -94,6 +94,8 @@ public:
return NKikimrViewer::EObjectType::Replication;
case NKikimrSchemeOp::EPathType::EPathTypeBlobDepot:
return NKikimrViewer::EObjectType::BlobDepot;
+ case NKikimrSchemeOp::EPathType::EPathTypeExternalTable:
+ return NKikimrViewer::EObjectType::ExternalTable;
case NKikimrSchemeOp::EPathType::EPathTypeExtSubDomain:
case NKikimrSchemeOp::EPathType::EPathTypeTableIndex:
case NKikimrSchemeOp::EPathType::EPathTypeInvalid:
diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto
index 07e2ecf75e..8190cff37a 100644
--- a/ydb/core/viewer/protos/viewer.proto
+++ b/ydb/core/viewer/protos/viewer.proto
@@ -35,6 +35,7 @@ enum EObjectType {
Sequence = 21;
Replication = 22;
BlobDepot = 23;
+ ExternalTable = 24;
}
message TBrowseInfo {
diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h
index d5e7ee1d64..1a079fc52b 100644
--- a/ydb/public/lib/deprecated/kicli/kicli.h
+++ b/ydb/public/lib/deprecated/kicli/kicli.h
@@ -580,6 +580,7 @@ public:
Sequence,
Replication,
BlobDepot,
+ ExternalTable
};
TSchemaObject(TSchemaObject&&) = default;
diff --git a/ydb/public/lib/deprecated/kicli/schema.cpp b/ydb/public/lib/deprecated/kicli/schema.cpp
index 421f4de33d..a32f82d403 100644
--- a/ydb/public/lib/deprecated/kicli/schema.cpp
+++ b/ydb/public/lib/deprecated/kicli/schema.cpp
@@ -122,6 +122,9 @@ void TSchemaObject::Drop() {
case EPathType::BlobDepot:
drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropBlobDepot);
break;
+ case EPathType::ExternalTable:
+ drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable);
+ break;
case EPathType::Unknown:
case EPathType::SubDomain:
case EPathType::RtmrVolume:
@@ -207,6 +210,8 @@ static TSchemaObject::EPathType GetType(const NKikimrSchemeOp::TDirEntry& entry)
return TSchemaObject::EPathType::Replication;
case NKikimrSchemeOp::EPathTypeBlobDepot:
return TSchemaObject::EPathType::BlobDepot;
+ case NKikimrSchemeOp::EPathTypeExternalTable:
+ return TSchemaObject::EPathType::ExternalTable;
case NKikimrSchemeOp::EPathTypeTableIndex:
case NKikimrSchemeOp::EPathTypeExtSubDomain:
case NKikimrSchemeOp::EPathTypeCdcStream:
diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
index 93f379652f..0c181e773f 100644
--- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
+++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
@@ -6770,5 +6770,77 @@
"Blobs": 1
}
}
+ },
+ {
+ "TableId": 104,
+ "TableName": "ExternalTable",
+ "TableKey": [
+ 1,
+ 2
+ ],
+ "ColumnsAdded": [
+ {
+ "ColumnId": 7,
+ "ColumnName": "Content",
+ "ColumnType": "String"
+ },
+ {
+ "ColumnId": 1,
+ "ColumnName": "OwnerPathId",
+ "ColumnType": "Uint64"
+ },
+ {
+ "ColumnId": 2,
+ "ColumnName": "LocalPathId",
+ "ColumnType": "Uint64"
+ },
+ {
+ "ColumnId": 3,
+ "ColumnName": "AlterVersion",
+ "ColumnType": "Uint64"
+ },
+ {
+ "ColumnId": 4,
+ "ColumnName": "SourceType",
+ "ColumnType": "Utf8"
+ },
+ {
+ "ColumnId": 5,
+ "ColumnName": "DataSourcePath",
+ "ColumnType": "Utf8"
+ },
+ {
+ "ColumnId": 6,
+ "ColumnName": "Location",
+ "ColumnType": "Utf8"
+ }
+ ],
+ "ColumnsDropped": [],
+ "ColumnFamilies": {
+ "0": {
+ "Columns": [
+ 7,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6
+ ],
+ "RoomID": 0,
+ "Codec": 0,
+ "InMemory": false,
+ "Cache": 0,
+ "Small": 4294967295,
+ "Large": 4294967295
+ }
+ },
+ "Rooms": {
+ "0": {
+ "Main": 1,
+ "Outer": 1,
+ "Blobs": 1
+ }
+ }
}
] \ No newline at end of file