diff options
author | hcpp <hcpp@ydb.tech> | 2023-02-08 13:20:50 +0300 |
---|---|---|
committer | hcpp <hcpp@ydb.tech> | 2023-02-08 13:20:50 +0300 |
commit | 6bf6928eda1dd0bcabb3aac0000b72aedd29d44e (patch) | |
tree | bbff709ccdc617317577432989f871a11f58a5ee | |
parent | 276e61af9d8785166ff14771a61e0acd8527b282 (diff) | |
download | ydb-6bf6928eda1dd0bcabb3aac0000b72aedd29d44e.tar.gz |
external table has been added to ss
47 files changed, 1993 insertions, 16 deletions
diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index 9c3ed4d879..cf5067a51c 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -173,6 +173,11 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxAlterExtSubDomainCreateHive = 140 [(CounterOpts) = {Name: "InFlightOps/AlterExtSubDomainCreateHive"}]; COUNTER_IN_FLIGHT_OPS_TxAlterCdcStreamAtTableDropSnapshot = 141 [(CounterOpts) = {Name: "InFlightOps/AlterCdcStreamAtTableDropSnapshot"}]; COUNTER_IN_FLIGHT_OPS_TxDropCdcStreamAtTableDropSnapshot = 142 [(CounterOpts) = {Name: "InFlightOps/DropCdcStreamAtTableDropSnapshot"}]; + + COUNTER_EXTERNAL_TABLE_COUNT = 143 [(CounterOpts) = {Name: "ExternalTableCount"}]; + COUNTER_IN_FLIGHT_OPS_TxCreateExternalTable = 144 [(CounterOpts) = {Name: "InFlightOps/CreateExternalTable"}]; + COUNTER_IN_FLIGHT_OPS_TxDropExternalTable = 145 [(CounterOpts) = {Name: "InFlightOps/DropExternalTable"}]; + COUNTER_IN_FLIGHT_OPS_TxAlterExternalTable = 146 [(CounterOpts) = {Name: "InFlightOps/AlterExternalTable"}]; } enum ECumulativeCounters { @@ -279,6 +284,10 @@ enum ECumulativeCounters { COUNTER_FINISHED_OPS_TxAlterExtSubDomainCreateHive = 85 [(CounterOpts) = {Name: "FinishedOps/AlterExtSubDomainCreateHive"}]; COUNTER_FINISHED_OPS_TxAlterCdcStreamAtTableDropSnapshot = 86 [(CounterOpts) = {Name: "FinishedOps/AlterCdcStreamAtTableDropSnapshot"}]; COUNTER_FINISHED_OPS_TxDropCdcStreamAtTableDropSnapshot = 87 [(CounterOpts) = {Name: "FinishedOps/DropCdcStreamAtTableDropSnapshot"}]; + + COUNTER_FINISHED_OPS_TxCreateExternalTable = 88 [(CounterOpts) = {Name: "FinishedOps/CreateExternalTable"}]; + COUNTER_FINISHED_OPS_TxDropExternalTable = 89 [(CounterOpts) = {Name: "FinishedOps/DropExternalTable"}]; + COUNTER_FINISHED_OPS_TxAlterExternalTable = 90 [(CounterOpts) = {Name: "FinishedOps/AlterExternalTable"}]; } enum EPercentileCounters { diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 81b930edb5..826e62b31f 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1297,6 +1297,11 @@ enum EOperationType { // AlterExtSubDomain suboperations ESchemeOpAlterExtSubDomainCreateHive = 85; + + // External Table + ESchemeOpCreateExternalTable = 86; + ESchemeOpDropExternalTable = 87; + ESchemeOpAlterExternalTable = 88; } message TApplyIf { @@ -1405,6 +1410,8 @@ message TModifyScheme { optional TPersQueueGroupAllocate AllocatePersQueueGroup = 56; optional TPersQueueGroupDeallocate DeallocatePersQueueGroup = 57; + + optional TExternalTableDescription CreateExternalTable = 58; } // "Script", used by client to parse text files with multiple DDL commands @@ -1460,6 +1467,7 @@ enum EPathType { EPathTypeSequence = 15; EPathTypeReplication = 16; EPathTypeBlobDepot = 17; + EPathTypeExternalTable = 18; } enum EPathSubType { @@ -1511,6 +1519,7 @@ message TPathVersion { optional uint64 SequenceVersion = 24; optional uint64 ReplicationVersion = 25; optional uint64 BlobDepotVersion = 26; + optional uint64 ExternalTableVersion = 27; } // Describes single path @@ -1596,6 +1605,7 @@ message TPathDescription { optional TSequenceDescription SequenceDescription = 24; optional TReplicationDescription ReplicationDescription = 25; optional TBlobDepotDescription BlobDepotDescription = 26; + optional TExternalTableDescription ExternalTableDescription = 27; } // For persisting AlterTable Tx description in Schemeshard internal DB @@ -1639,3 +1649,14 @@ message TResourceProfile { // Resource Broker task type for large transactions. optional string LargeTxTaskType = 12 [default = "transaction"]; } + +message TExternalTableDescription { + optional string Name = 1; + optional NKikimrProto.TPathID PathId = 2; + optional uint64 Version = 3; + optional string SourceType = 4; + optional string DataSourcePath = 5; + optional string Location = 6; + repeated TColumnDescription Columns = 7; + optional bytes Content = 8; +} diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index 8955be17b5..b0c7b7fe85 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -736,6 +736,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { SequenceInfo.Drop(); ReplicationInfo.Drop(); BlobDepotInfo.Drop(); + ExternalTableInfo.Drop(); } void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) { @@ -1148,8 +1149,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { root.WriteKey(#name).UnsafeWriteValue(ProtoJsonString(name->Description)); \ } - DESCRIPTION_PART(DomainDescription) - DESCRIPTION_PART(RtmrVolumeInfo) + DESCRIPTION_PART(DomainDescription); + DESCRIPTION_PART(RtmrVolumeInfo); DESCRIPTION_PART(KesusInfo); DESCRIPTION_PART(SolomonVolumeInfo); DESCRIPTION_PART(PQGroupInfo); @@ -1159,6 +1160,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { DESCRIPTION_PART(SequenceInfo); DESCRIPTION_PART(ReplicationInfo); DESCRIPTION_PART(BlobDepotInfo); + DESCRIPTION_PART(ExternalTableInfo); #undef DESCRIPTION_PART @@ -1465,6 +1467,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Kind = TNavigate::KindBlobDepot; FillInfo(Kind, BlobDepotInfo, std::move(*pathDesc.MutableBlobDepotDescription())); break; + case NKikimrSchemeOp::EPathTypeExternalTable: + Kind = TNavigate::KindExternalTable; + FillInfo(Kind, ExternalTableInfo, std::move(*pathDesc.MutableExternalTableDescription())); + break; case NKikimrSchemeOp::EPathTypeInvalid: Y_VERIFY_DEBUG(false, "Invalid path type"); break; @@ -1522,6 +1528,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { case NKikimrSchemeOp::EPathTypeBlobDepot: ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindBlobDepot); break; + case NKikimrSchemeOp::EPathTypeExternalTable: + ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindExternalTable); + break; case NKikimrSchemeOp::EPathTypeTableIndex: case NKikimrSchemeOp::EPathTypeInvalid: Y_VERIFY_DEBUG(false, "Invalid path type"); @@ -1730,6 +1739,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.SequenceInfo = SequenceInfo; entry.ReplicationInfo = ReplicationInfo; entry.BlobDepotInfo = BlobDepotInfo; + entry.ExternalTableInfo = ExternalTableInfo; } bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry, @@ -2004,6 +2014,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { // BlobDepot specific TIntrusivePtr<TNavigate::TBlobDepotInfo> BlobDepotInfo; + // ExternalTable specific + TIntrusivePtr<TNavigate::TExternalTableInfo> ExternalTableInfo; + }; // TCacheItem struct TMerger { diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index 4786dd0887..913cc74f7e 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -129,6 +129,7 @@ struct TSchemeCacheNavigate { KindSequence = 14, KindReplication = 15, KindBlobDepot = 16, + KindExternalTable = 17, }; struct TListNodeEntry : public TAtomicRefCount<TListNodeEntry> { @@ -211,6 +212,11 @@ struct TSchemeCacheNavigate { NKikimrSchemeOp::TBlobDepotDescription Description; }; + struct TExternalTableInfo : public TAtomicRefCount<TExternalTableInfo> { + EKind Kind = KindUnknown; + NKikimrSchemeOp::TExternalTableDescription Description; + }; + struct TEntry { enum class ERequestType : ui8 { ByPath, @@ -256,6 +262,7 @@ struct TSchemeCacheNavigate { TIntrusiveConstPtr<TSequenceInfo> SequenceInfo; TIntrusiveConstPtr<TReplicationInfo> ReplicationInfo; TIntrusiveConstPtr<TBlobDepotInfo> BlobDepotInfo; + TIntrusiveConstPtr<TExternalTableInfo> ExternalTableInfo; TString ToString() const; TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; diff --git a/ydb/core/tx/schemeshard/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/CMakeLists.darwin.txt index a6d51a786d..9db71ca37f 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.darwin.txt @@ -17,6 +17,8 @@ add_subdirectory(ut_cdc_stream_reboots) add_subdirectory(ut_compaction) add_subdirectory(ut_export) add_subdirectory(ut_export_reboots_s3) +add_subdirectory(ut_external_table) +add_subdirectory(ut_external_table_reboots) add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) @@ -136,6 +138,7 @@ target_sources(core-tx-schemeshard PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_fs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp @@ -153,6 +156,7 @@ target_sources(core-tx-schemeshard PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_extsubdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_fs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt index 5ed31fe71d..49c3a0454e 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt @@ -17,6 +17,8 @@ add_subdirectory(ut_cdc_stream_reboots) add_subdirectory(ut_compaction) add_subdirectory(ut_export) add_subdirectory(ut_export_reboots_s3) +add_subdirectory(ut_external_table) +add_subdirectory(ut_external_table_reboots) add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) @@ -137,6 +139,7 @@ target_sources(core-tx-schemeshard PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_fs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp @@ -154,6 +157,7 @@ target_sources(core-tx-schemeshard PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_extsubdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_fs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/CMakeLists.linux.txt index 5ed31fe71d..49c3a0454e 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux.txt @@ -17,6 +17,8 @@ add_subdirectory(ut_cdc_stream_reboots) add_subdirectory(ut_compaction) add_subdirectory(ut_export) add_subdirectory(ut_export_reboots_s3) +add_subdirectory(ut_external_table) +add_subdirectory(ut_external_table_reboots) add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) @@ -137,6 +139,7 @@ target_sources(core-tx-schemeshard PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_bsv.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_extsubdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_fs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_index.cpp @@ -154,6 +157,7 @@ target_sources(core-tx-schemeshard PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_subdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_bsv.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_extsubdomain.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_fs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/schemeshard__operation_drop_indexed_table.cpp diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 9b3ea10a73..d0a0736874 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1877,6 +1877,30 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { } + // Read External Tables + { + auto rowset = db.Table<Schema::ExternalTable>().Range().Select(); + if (!rowset.IsReady()) + return false; + + while (!rowset.EndOfSet()) { + TOwnerId ownerPathId = rowset.GetValue<Schema::ExternalTable::OwnerPathId>(); + TLocalPathId localPathId = rowset.GetValue<Schema::ExternalTable::LocalPathId>(); + TPathId pathId(ownerPathId, localPathId); + + auto& externalTable = Self->ExternalTables[pathId] = new TExternalTableInfo(); + externalTable->SourceType = rowset.GetValue<Schema::ExternalTable::SourceType>(); + externalTable->DataSourcePath = rowset.GetValue<Schema::ExternalTable::DataSourcePath>(); + externalTable->Location = rowset.GetValue<Schema::ExternalTable::Location>(); + externalTable->AlterVersion = rowset.GetValue<Schema::ExternalTable::AlterVersion>(); + externalTable->Content = rowset.GetValue<Schema::ExternalTable::Content>(); + Self->IncrementPathDbRefCount(pathId); + + if (!rowset.Next()) + return false; + } + } + // Read table columns { TColumnRows columnRows; @@ -1903,13 +1927,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto notNull = std::get<10>(rec); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); - Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable(), "Path is not a table, pathId: " << pathId); - Y_VERIFY_S(Self->Tables.FindPtr(pathId), "Table doesn't exist, pathId: " << pathId); - - TTableInfo::TPtr tableInfo = Self->Tables[pathId]; - Y_VERIFY_S(colId < tableInfo->NextColumnId, "Column id should be less than NextColId" - << ", columnId: " << colId - << ", NextColId: " << tableInfo->NextColumnId); + Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable() || Self->PathsById.at(pathId)->IsExternalTable(), "Path is not a table or external table, pathId: " << pathId); + Y_VERIFY_S(Self->Tables.FindPtr(pathId) || Self->ExternalTables.FindPtr(pathId), "Table or external table don't exist, pathId: " << pathId); TTableInfo::TColumn colInfo(colName, colId, typeInfo); colInfo.KeyOrder = keyOrder; @@ -1920,11 +1939,21 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { colInfo.DefaultValue = defaultValue; colInfo.NotNull = notNull; - tableInfo->Columns[colId] = colInfo; + if (auto it = Self->Tables.find(pathId); it != Self->Tables.end()) { + TTableInfo::TPtr tableInfo = it->second; + Y_VERIFY_S(colId < tableInfo->NextColumnId, "Column id should be less than NextColId" + << ", columnId: " << colId + << ", NextColId: " << tableInfo->NextColumnId); - if (colInfo.KeyOrder != (ui32)-1) { - tableInfo->KeyColumnIds.resize(Max<ui32>(tableInfo->KeyColumnIds.size(), colInfo.KeyOrder + 1)); - tableInfo->KeyColumnIds[colInfo.KeyOrder] = colId; + tableInfo->Columns[colId] = colInfo; + + if (colInfo.KeyOrder != (ui32)-1) { + tableInfo->KeyColumnIds.resize(Max<ui32>(tableInfo->KeyColumnIds.size(), colInfo.KeyOrder + 1)); + tableInfo->KeyColumnIds[colInfo.KeyOrder] = colId; + } + } else if (auto it = Self->ExternalTables.find(pathId); it != Self->ExternalTables.end()) { + TExternalTableInfo::TPtr externalTableInfo = it->second; + externalTableInfo->Columns[colId] = colInfo; } } } @@ -3945,6 +3974,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Self->TabletCounters->Simple()[COUNTER_SEQUENCE_COUNT].Add(1); } else if (path->IsReplication()) { Self->TabletCounters->Simple()[COUNTER_REPLICATION_COUNT].Add(1); + } else if (path->IsExternalTable()) { + Self->TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Add(1); } path->ApplySpecialAttributes(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp index bb83d34e90..4d7fccde2b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp @@ -666,6 +666,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable: targetName = tx.GetCreateColumnTable().GetName(); break; + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable: + targetName = tx.GetCreateExternalTable().GetName(); + break; default: result.Transactions.push_back(tx); return result; @@ -760,6 +763,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx case NKikimrSchemeOp::EOperationType::ESchemeOpCreateColumnTable: create.MutableCreateColumnTable()->SetName(name); break; + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable: + create.MutableCreateExternalTable()->SetName(name); + break; default: Y_UNREACHABLE(); } @@ -1004,7 +1010,12 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta return CreateAlterBlobDepot(NextPartId(), txState); case TTxState::ETxType::TxDropBlobDepot: return CreateDropBlobDepot(NextPartId(), txState); - + case TTxState::ETxType::TxCreateExternalTable: + return CreateNewExternalTable(NextPartId(), txState); + case TTxState::ETxType::TxDropExternalTable: + return CreateDropExternalTable(NextPartId(), txState); + case TTxState::ETxType::TxAlterExternalTable: + Y_FAIL("TODO: implement"); case TTxState::ETxType::TxInvalid: Y_UNREACHABLE(); } @@ -1205,6 +1216,14 @@ ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationTyp return CreateAlterBlobDepot(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropBlobDepot: return CreateDropBlobDepot(NextPartId(), tx); + + // ExternalTable + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable: + return CreateNewExternalTable(NextPartId(), tx); + case NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable: + return CreateDropExternalTable(NextPartId(), tx); + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExternalTable: + Y_FAIL("TODO: implement"); } Y_UNREACHABLE(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp new file mode 100644 index 0000000000..7621265af9 --- /dev/null +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp @@ -0,0 +1,344 @@ +#include "schemeshard__operation_part.h" +#include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" + +#include <ydb/core/base/subdomain.h> + +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + + +namespace { + +using namespace NKikimr; +using namespace NSchemeShard; + +bool IsAllowedType(ui32 typeId) { + if (!NScheme::NTypeIds::IsYqlType(typeId)) { + return false; + } + + switch (typeId) { + case NYql::NProto::Bool: + case NYql::NProto::Interval: + case NYql::NProto::Decimal: + case NYql::NProto::DyNumber: + return false; + default: + break; + } + return true; +} + +TExternalTableInfo::TPtr CreateExternalTable(const NKikimrSchemeOp::TExternalTableDescription& desc, TString& errStr) { + TExternalTableInfo::TPtr externalTableInfo = new TExternalTableInfo; + const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; + + externalTableInfo->DataSourcePath = desc.GetDataSourcePath(); + externalTableInfo->Location = desc.GetLocation(); + externalTableInfo->AlterVersion = 1; + externalTableInfo->Content = desc.GetContent(); + + uint64_t nextColumnId = 1; + for (const auto& col : desc.GetColumns()) { + TString colName = col.GetName(); + + if (!colName) { + errStr = "Columns cannot have an empty name"; + return nullptr; + } + + if (col.HasTypeId()) { + errStr = TStringBuilder() << "Cannot set TypeId for column '" << colName << "', use Type"; + return nullptr; + } + + if (!col.HasType()) { + errStr = TStringBuilder() << "Missing Type for column '" << colName << "'"; + return nullptr; + } + + auto typeName = NMiniKQL::AdaptLegacyYqlType(col.GetType()); + const NScheme::IType* type = typeRegistry->GetType(typeName); + + if (!type || !IsAllowedType(type->GetTypeId())) { + errStr = TStringBuilder() + << "Type '" << col.GetType() << "' specified for column '" << colName << "' is not supported"; + return nullptr; + } + + NScheme::TTypeInfo typeInfo; + if (type) { + // Only allow YQL types + if (!NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { + errStr = Sprintf("Type '%s' specified for column '%s' is no longer supported", col.GetType().data(), colName.data()); + return nullptr; + } + typeInfo = NScheme::TTypeInfo(type->GetTypeId()); + } else { + auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName); + if (!typeDesc) { + errStr = Sprintf("Type '%s' specified for column '%s' is not supported by storage", col.GetType().data(), colName.data()); + return nullptr; + } + typeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); + } + + ui32 colId = col.HasId() ? col.GetId() : nextColumnId; + if (externalTableInfo->Columns.contains(colId)) { + errStr = Sprintf("Duplicate column id: %" PRIu32, colId); + return nullptr; + } + + nextColumnId = colId + 1 > nextColumnId ? colId + 1 : nextColumnId; + + TTableInfo::TColumn& column = externalTableInfo->Columns[colId]; + column = TTableInfo::TColumn(colName, colId, typeInfo); + column.NotNull = col.GetNotNull(); + } + + return externalTableInfo; +} + +class TPropose: public TSubOperationState { +private: + TOperationId OperationId; + + TString DebugHint() const override { + return TStringBuilder() + << "TCreateExternalTable TPropose" + << ", operationId: " << OperationId; + } + +public: + TPropose(TOperationId id) + : OperationId(id) + { + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const TStepId step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + TTxState* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateExternalTable); + + auto pathId = txState->TargetPathId; + auto path = TPath::Init(pathId, context.SS); + TPathElement::TPtr pathPtr = context.SS->PathsById.at(pathId); + + context.SS->TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Add(1); + + NIceDb::TNiceDb db(context.GetDB()); + + path->StepCreated = step; + context.SS->PersistCreateStep(db, pathId, step); + + IncParentDirAlterVersionWithRepublish(OperationId, path, context); + + context.SS->ClearDescribePathCaches(pathPtr); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + return true; + } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + TTxState* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxCreateExternalTable); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } +}; + + +class TCreateExternalTable: public TSubOperation { + static TTxState::ETxState NextState() { + return TTxState::Propose; + } + + TTxState::ETxState NextState(TTxState::ETxState state) const override { + switch (state) { + case TTxState::Waiting: + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override { + switch (state) { + case TTxState::Waiting: + case TTxState::Propose: + return MakeHolder<TPropose>(OperationId); + case TTxState::Done: + return MakeHolder<TDone>(OperationId); + default: + return nullptr; + } + } + +public: + using TSubOperation::TSubOperation; + + THolder<TProposeResponse> Propose(const TString& owner, TOperationContext& context) override { + const auto ssId = context.SS->SelfTabletId(); + + const auto acceptExisted = !Transaction.GetFailOnExist(); + const TString& parentPathStr = Transaction.GetWorkingDir(); + const auto& externalTableDescription = Transaction.GetCreateExternalTable(); + const TString& name = externalTableDescription.GetName(); + + + LOG_N("TCreateExternalTable Propose" + << ": opId# " << OperationId + << ", path# " << parentPathStr << "/" << name); + + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId)); + + NSchemeShard::TPath parentPath = NSchemeShard::TPath::Resolve(parentPathStr, context.SS); + { + NSchemeShard::TPath::TChecker checks = parentPath.Check(); + checks + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsCommonSensePath() + .IsLikeDirectory(); + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + return result; + } + } + + const TString acl = Transaction.GetModifyACL().GetDiffACL(); + + NSchemeShard::TPath dstPath = parentPath.Child(name); + { + NSchemeShard::TPath::TChecker checks = dstPath.Check(); + checks.IsAtLocalSchemeShard(); + if (dstPath.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() + .FailOnExist(TPathElement::EPathType::EPathTypeExternalTable, acceptExisted); + } else { + checks + .NotEmpty() + .NotResolved(); + } + + if (checks) { + checks + .IsValidLeafName() + .DepthLimit() + .PathsLimit() + .DirChildrenLimit() + .IsValidACL(acl); + } + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + if (dstPath.IsResolved()) { + result->SetPathCreateTxId(ui64(dstPath.Base()->CreateTxId)); + result->SetPathId(dstPath.Base()->PathId.LocalPathId); + } + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); + return result; + } + + TExternalTableInfo::TPtr externalTableInfo = CreateExternalTable(externalTableDescription, errStr); + if (!externalTableInfo) { + result->SetError(NKikimrScheme::StatusSchemeError, errStr); + return result; + } + + dstPath.MaterializeLeaf(owner); + result->SetPathId(dstPath.Base()->PathId.LocalPathId); + + TPathElement::TPtr externalTable = dstPath.Base(); + externalTable->CreateTxId = OperationId.GetTxId(); + externalTable->LastTxId = OperationId.GetTxId(); + externalTable->PathState = TPathElement::EPathState::EPathStateCreate; + externalTable->PathType = TPathElement::EPathType::EPathTypeExternalTable; + + TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateExternalTable, externalTable->PathId); + txState.Shards.clear(); + + NIceDb::TNiceDb db(context.GetDB()); + + if (parentPath.Base()->HasActiveChanges()) { + TTxId parentTxId = parentPath.Base()->PlannedToCreate() ? parentPath.Base()->CreateTxId : parentPath.Base()->LastTxId; + context.OnComplete.Dependence(parentTxId, OperationId.GetTxId()); + } + + context.SS->ChangeTxState(db, OperationId, TTxState::Propose); + context.OnComplete.ActivateTx(OperationId); + + context.SS->ExternalTables[externalTable->PathId] = externalTableInfo; + context.SS->IncrementPathDbRefCount(externalTable->PathId); + + context.SS->PersistPath(db, externalTable->PathId); + + if (!acl.empty()) { + externalTable->ApplyACL(acl); + context.SS->PersistACL(db, externalTable); + } + + context.SS->PersistExternalTable(db, externalTable->PathId, externalTableInfo); + context.SS->PersistTxState(db, OperationId); + + IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, dstPath, context.SS, context.OnComplete); + + dstPath.DomainInfo()->IncPathsInside(); + parentPath.Base()->IncAliveChildren(); + + SetState(NextState()); + return result; + } + + void AbortPropose(TOperationContext& context) override { + LOG_N("TCreateExternalTable AbortPropose" + << ": opId# " << OperationId); + Y_FAIL("no AbortPropose for TCreateExternalTable"); + } + + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { + LOG_N("TCreateExternalTable AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << forceDropTxId); + context.OnComplete.DoneOperation(OperationId); + } +}; + +} + +namespace NKikimr::NSchemeShard { + +ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, const TTxTransaction& tx) { + return MakeSubOperation<TCreateExternalTable>(id, tx); +} + +ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, TTxState::ETxState state) { + Y_VERIFY(state != TTxState::Invalid); + return MakeSubOperation<TCreateExternalTable>(id, state); +} + +} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp new file mode 100644 index 0000000000..cea1bf6ccb --- /dev/null +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_table.cpp @@ -0,0 +1,209 @@ +#include "schemeshard__operation_common.h" +#include "schemeshard__operation_part.h" +#include "schemeshard_impl.h" + +#define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) + +namespace { + +using namespace NKikimr; +using namespace NSchemeShard; + +class TPropose: public TSubOperationState { + TString DebugHint() const override { + return TStringBuilder() + << "TDropExternalTable TPropose" + << " opId# " << OperationId << " "; + } + +public: + explicit TPropose(TOperationId id) + : OperationId(id) + { } + + bool ProgressState(TOperationContext& context) override { + LOG_I(DebugHint() << "ProgressState"); + + const auto* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + Y_VERIFY(txState->TxType == TTxState::TxDropExternalTable); + + context.OnComplete.ProposeToCoordinator(OperationId, txState->TargetPathId, TStepId(0)); + return false; + } + + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { + const auto step = TStepId(ev->Get()->StepId); + + LOG_I(DebugHint() << "HandleReply TEvOperationPlan" + << ": step# " << step); + + NIceDb::TNiceDb db(context.GetDB()); + + TTxState* txState = context.SS->FindTx(OperationId); + Y_VERIFY(txState); + TPathId pathId = txState->TargetPathId; + auto path = context.SS->PathsById.at(pathId); + auto parentDir = context.SS->PathsById.at(path->ParentPathId); + + Y_VERIFY(!path->Dropped()); + path->SetDropped(step, OperationId.GetTxId()); + context.SS->PersistDropStep(db, pathId, step, OperationId); + auto domainInfo = context.SS->ResolveDomainInfo(pathId); + domainInfo->DecPathsInside(); + parentDir->DecAliveChildren(); + + context.SS->TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Sub(1); + context.SS->PersistRemoveExternalTable(db, pathId); + + ++parentDir->DirAlterVersion; + context.SS->PersistPathDirAlterVersion(db, parentDir); + context.SS->ClearDescribePathCaches(parentDir); + context.SS->ClearDescribePathCaches(path); + + if (!context.SS->DisablePublicationsOfDropping) { + context.OnComplete.PublishToSchemeBoard(OperationId, parentDir->PathId); + context.OnComplete.PublishToSchemeBoard(OperationId, pathId); + } + + context.SS->ChangeTxState(db, OperationId, TTxState::Done); + + return true; + } + +private: + const TOperationId OperationId; + +}; // TPropose + +class TDropExternalTable: public TSubOperation { + TTxState::ETxState NextState() const { + return TTxState::Propose; + } + + TTxState::ETxState NextState(TTxState::ETxState state) const override { + switch (state) { + case TTxState::Propose: + return TTxState::Done; + default: + return TTxState::Invalid; + } + } + + TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override { + switch (state) { + case TTxState::Propose: + return MakeHolder<TPropose>(OperationId); + case TTxState::Done: + return MakeHolder<TDone>(OperationId); + default: + return nullptr; + } + } + +public: + using TSubOperation::TSubOperation; + + THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { + const ui64 ssId = context.SS->TabletID(); + const auto& drop = Transaction.GetDrop(); + + const TString& workingDir = Transaction.GetWorkingDir(); + const TString& name = drop.GetName(); + + LOG_N("TDropExternalTable Propose" + << ": opId# " << OperationId + << ", path# " << workingDir << "/" << name); + + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ssId); + + TPath path = drop.HasId() + ? TPath::Init(context.SS->MakeLocalId(drop.GetId()), context.SS) + : TPath::Resolve(workingDir, context.SS).Dive(name); + { + TPath::TChecker checks = path.Check(); + checks + .NotEmpty() + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved() + .NotDeleted() + .NotUnderDeleting() + .IsExternalTable() + .NotUnderOperation() + .IsCommonSensePath(); + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + if (path.IsResolved() && path.Base()->IsExternalTable() && (path.Base()->PlannedToDrop() || path.Base()->Dropped())) { + result->SetPathDropTxId(ui64(path.Base()->DropTxId)); + result->SetPathId(path.Base()->PathId.LocalPathId); + } + return result; + } + } + + TString errStr; + if (!context.SS->CheckApplyIf(Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); + return result; + } + + const auto pathId = path.Base()->PathId; + result->SetPathId(pathId.LocalPathId); + + auto guard = context.DbGuard(); + context.MemChanges.GrabNewTxState(context.SS, OperationId); + context.MemChanges.GrabPath(context.SS, pathId); + context.MemChanges.GrabPath(context.SS, path->ParentPathId); + context.MemChanges.GrabExternalTable(context.SS, pathId); + + context.DbChanges.PersistTxState(OperationId); + context.DbChanges.PersistPath(pathId); + context.DbChanges.PersistPath(path->ParentPathId); + + Y_VERIFY(!context.SS->FindTx(OperationId)); + TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxDropExternalTable, path.Base()->PathId); + txState.State = TTxState::Propose; + txState.MinStep = TStepId(1); + + path.Base()->PathState = TPathElement::EPathState::EPathStateDrop; + path.Base()->DropTxId = OperationId.GetTxId(); + path.Base()->LastTxId = OperationId.GetTxId(); + + IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, path, context.SS, context.OnComplete); + + context.OnComplete.ActivateTx(OperationId); + + SetState(NextState()); + return result; + } + + void AbortPropose(TOperationContext& context) override { + LOG_N("TDropExternalTable AbortPropose" + << ": opId# " << OperationId); + } + + void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { + LOG_N("TDropExternalTable AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << forceDropTxId); + context.OnComplete.DoneOperation(OperationId); + } +}; + +} + +namespace NKikimr::NSchemeShard { + +ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, const TTxTransaction& tx) { + return MakeSubOperation<TDropExternalTable>(id, tx); +} + +ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, TTxState::ETxState state) { + Y_VERIFY(state != TTxState::Invalid); + return MakeSubOperation<TDropExternalTable>(id, state); +} + +} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp index 893ecaeefa..da529e7c35 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.cpp @@ -82,6 +82,14 @@ void TMemoryChanges::GrabLongLock(TSchemeShard* ss, const TPathId& pathId, TTxId LockedPaths.emplace(pathId, lockTxId); // will be restored on UnDo() } +void TMemoryChanges::GrabNewExternalTable(TSchemeShard* ss, const TPathId& pathId) { + GrabNew(pathId, ss->ExternalTables, ExternalTables); +} + +void TMemoryChanges::GrabExternalTable(TSchemeShard* ss, const TPathId& pathId) { + Grab<TExternalTableInfo>(pathId, ss->ExternalTables, ExternalTables); +} + void TMemoryChanges::UnDo(TSchemeShard* ss) { // be aware of the order of grab & undo ops // stack is the best way to manage it right @@ -181,6 +189,16 @@ void TMemoryChanges::UnDo(TSchemeShard* ss) { } TxStates.pop(); } + + while (ExternalTables) { + const auto& [id, elem] = ExternalTables.top(); + if (elem) { + ss->ExternalTables[id] = elem; + } else { + ss->ExternalTables.erase(id); + } + ExternalTables.pop(); + } } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h index f5b73942c4..6c9032b12b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_memory_changes.h @@ -39,6 +39,9 @@ class TMemoryChanges: public TSimpleRefCount<TMemoryChanges> { using TTxState = std::pair<TOperationId, THolder<TTxState>>; TStack<TTxState> TxStates; + using TExternalTableState = std::pair<TPathId, TExternalTableInfo::TPtr>; + TStack<TExternalTableState> ExternalTables; + public: ~TMemoryChanges() = default; @@ -66,6 +69,9 @@ public: void GrabNewLongLock(TSchemeShard* ss, const TPathId& pathId); void GrabLongLock(TSchemeShard* ss, const TPathId& pathId, TTxId lockTxId); + void GrabNewExternalTable(TSchemeShard* ss, const TPathId& pathId); + void GrabExternalTable(TSchemeShard* ss, const TPathId& pathId); + void UnDo(TSchemeShard* ss); }; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index 41526b52f5..b6e0d7eba4 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -369,6 +369,14 @@ ISubOperationBase::TPtr CreateDropTableIndexAtMainTable(TOperationId id, TTxStat ISubOperationBase::TPtr CreateUpdateMainTableOnIndexMove(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateUpdateMainTableOnIndexMove(TOperationId id, TTxState::ETxState state); +// External Table +// Create +ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateNewExternalTable(TOperationId id, TTxState::ETxState state); +// Drop +ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateDropExternalTable(TOperationId id, TTxState::ETxState state); + /// CDC // Create TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId id, const TTxTransaction& tx, TOperationContext& context); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp index 7bb8f2fe8f..9f67b76510 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp @@ -303,6 +303,7 @@ public: auto migrateShards = event->Record.MutableShards(); switch (path.Base()->PathType) { case NKikimrSchemeOp::EPathType::EPathTypeDir: + case NKikimrSchemeOp::EPathType::EPathTypeExternalTable: Y_VERIFY(!path.Base()->IsRoot()); //no shards break; diff --git a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp index 9b4d522f15..a65626eef2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp @@ -186,6 +186,12 @@ TString DefineUserOperationName(NKikimrSchemeOp::EOperationType type) { return "ALTER BLOB DEPOT"; case NKikimrSchemeOp::EOperationType::ESchemeOpDropBlobDepot: return "DROP BLOB DEPOT"; + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable: + return "CREATE EXTERNAL TABLE"; + case NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable: + return "DROP EXTERNAL TABLE"; + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExternalTable: + return "ALTER EXTERNAL TABLE"; } Y_FAIL("switch should cover all operation types"); } @@ -445,6 +451,15 @@ TVector<TString> ExtractChangingPaths(const NKikimrSchemeOp::TModifyScheme& tx) result.emplace_back(NKikimr::JoinPath({tx.GetMoveIndex().GetTablePath(), tx.GetMoveIndex().GetSrcPath()})); result.emplace_back(NKikimr::JoinPath({tx.GetMoveIndex().GetTablePath(), tx.GetMoveIndex().GetDstPath()})); break; + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable: + result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetCreateExternalTable().GetName()})); + break; + case NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable: + result.emplace_back(NKikimr::JoinPath({tx.GetWorkingDir(), tx.GetDrop().GetName()})); + break; + case NKikimrSchemeOp::EOperationType::ESchemeOpAlterExternalTable: + // TODO: unimplemented + break; } return result; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 5804e7c5b6..117351ed00 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -1373,6 +1373,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxCreateSequence: case TTxState::TxCreateReplication: case TTxState::TxCreateBlobDepot: + case TTxState::TxCreateExternalTable: return TPathElement::EPathState::EPathStateCreate; case TTxState::TxAlterPQGroup: case TTxState::TxAlterTable: @@ -1403,6 +1404,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxAlterReplication: case TTxState::TxAlterBlobDepot: case TTxState::TxUpdateMainTableOnIndexMove: + case TTxState::TxAlterExternalTable: return TPathElement::EPathState::EPathStateAlter; case TTxState::TxDropTable: case TTxState::TxDropPQGroup: @@ -1421,6 +1423,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxDropSequence: case TTxState::TxDropReplication: case TTxState::TxDropBlobDepot: + case TTxState::TxDropExternalTable: return TPathElement::EPathState::EPathStateDrop; case TTxState::TxBackup: return TPathElement::EPathState::EPathStateBackup; @@ -2588,6 +2591,56 @@ void TSchemeShard::PersistRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId, const } } +void TSchemeShard::PersistExternalTable(NIceDb::TNiceDb &db, TPathId pathId, const TExternalTableInfo::TPtr externalTableInfo) { + Y_VERIFY(IsLocalId(pathId)); + + db.Table<Schema::ExternalTable>().Key(pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate<Schema::ExternalTable::SourceType>{externalTableInfo->SourceType}, + NIceDb::TUpdate<Schema::ExternalTable::DataSourcePath>{externalTableInfo->DataSourcePath}, + NIceDb::TUpdate<Schema::ExternalTable::Location>{externalTableInfo->Location}, + NIceDb::TUpdate<Schema::ExternalTable::AlterVersion>{externalTableInfo->AlterVersion}, + NIceDb::TUpdate<Schema::ExternalTable::Content>{externalTableInfo->Content}); + + for (auto col : externalTableInfo->Columns) { + ui32 colId = col.first; + const TTableInfo::TColumn& cinfo = col.second; + TString typeData; + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + if (columnType.TypeInfo) { + Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData)); + } + db.Table<Schema::MigratedColumns>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Update( + NIceDb::TUpdate<Schema::MigratedColumns::ColName>(cinfo.Name), + NIceDb::TUpdate<Schema::MigratedColumns::ColType>((ui32)columnType.TypeId), + NIceDb::TUpdate<Schema::MigratedColumns::ColTypeData>(typeData), + NIceDb::TUpdate<Schema::MigratedColumns::ColKeyOrder>(cinfo.KeyOrder), + NIceDb::TUpdate<Schema::MigratedColumns::CreateVersion>(cinfo.CreateVersion), + NIceDb::TUpdate<Schema::MigratedColumns::DeleteVersion>(cinfo.DeleteVersion), + NIceDb::TUpdate<Schema::MigratedColumns::Family>(cinfo.Family), + NIceDb::TUpdate<Schema::MigratedColumns::DefaultKind>(cinfo.DefaultKind), + NIceDb::TUpdate<Schema::MigratedColumns::DefaultValue>(cinfo.DefaultValue), + NIceDb::TUpdate<Schema::MigratedColumns::NotNull>(cinfo.NotNull)); + } +} + +void TSchemeShard::PersistRemoveExternalTable(NIceDb::TNiceDb& db, TPathId pathId) +{ + Y_VERIFY(IsLocalId(pathId)); + if (ExternalTables.contains(pathId)) { + auto externalTableInfo = ExternalTables.at(pathId); + + for (auto col : externalTableInfo->Columns) { + const ui32 colId = col.first; + db.Table<Schema::MigratedColumns>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Delete(); + } + + ExternalTables.erase(pathId); + DecrementPathDbRefCount(pathId); + } + + db.Table<Schema::ExternalTable>().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); +} + void TSchemeShard::PersistRemoveRtmrVolume(NIceDb::TNiceDb &db, TPathId pathId) { Y_VERIFY(IsLocalId(pathId)); @@ -3819,6 +3872,13 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co Y_FAIL_S("BlobDepot for path " << pathId << " not found"); } break; + case NKikimrSchemeOp::EPathType::EPathTypeExternalTable: { + auto it = ExternalTables.find(pathId); + Y_VERIFY(it != ExternalTables.end()); + result.SetExternalTableVersion(it->second->AlterVersion); + generalVersion += result.GetExternalTableVersion(); + break; + } case NKikimrSchemeOp::EPathType::EPathTypeInvalid: { Y_UNREACHABLE(); @@ -4567,6 +4627,9 @@ void TSchemeShard::UncountNode(TPathElement::TPtr node) { case TPathElement::EPathType::EPathTypeBlobDepot: TabletCounters->Simple()[COUNTER_BLOB_DEPOT_COUNT].Sub(1); break; + case TPathElement::EPathType::EPathTypeExternalTable: + TabletCounters->Simple()[COUNTER_EXTERNAL_TABLE_COUNT].Sub(1); + break; case TPathElement::EPathType::EPathTypeInvalid: Y_FAIL("impossible path type"); } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index b6b276f1ae..37c0803cd5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -209,6 +209,7 @@ public: THashMap<TPathId, TFileStoreInfo::TPtr> FileStoreInfos; THashMap<TPathId, TKesusInfo::TPtr> KesusInfos; THashMap<TPathId, TOlapStoreInfo::TPtr> OlapStores; + THashMap<TPathId, TExternalTableInfo::TPtr> ExternalTables; TTablesStorage ColumnTables; @@ -729,6 +730,10 @@ public: void PersistStorageBillingTime(NIceDb::TNiceDb& db); + // ExternalTable + void PersistExternalTable(NIceDb::TNiceDb &db, TPathId pathId, const TExternalTableInfo::TPtr externalTable); + void PersistRemoveExternalTable(NIceDb::TNiceDb& db, TPathId pathId); + TTabletId GetGlobalHive(const TActorContext& ctx) const; enum class EHiveSelection : uint8_t { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 304b441cd1..b94a63e9a1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2897,6 +2897,17 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> { NKikimrSchemeOp::TIndexBuildConfig SerializeToProto(TSchemeShard* ss) const; }; +struct TExternalTableInfo: TSimpleRefCount<TExternalTableInfo> { + using TPtr = TIntrusivePtr<TExternalTableInfo>; + + TString SourceType; + TString DataSourcePath; + TString Location; + ui64 AlterVersion = 0; + THashMap<ui32, TTableInfo::TColumn> Columns; + TString Content; +}; + bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, const THashMap<ui32, TTableInfo::TColumn>& sourceColumns, const THashMap<ui32, TTableInfo::TColumn>& alterColumns, diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index b94a2fb456..c68fbbec34 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -742,6 +742,19 @@ const TPath::TChecker& TPath::TChecker::PQReservedStorageLimit(ui64 delta, EStat return *this; } +const TPath::TChecker& TPath::TChecker::IsExternalTable(EStatus status) const { + if (Failed) { + return *this; + } + + if (Path.Base()->IsExternalTable()) { + return *this; + } + + return Fail(status, TStringBuilder() << "path is not a external table" + << " (" << BasicPathInfo(Path.Base()) << ")"); +} + const TPath::TChecker& TPath::TChecker::PathShardsLimit(ui64 delta, EStatus status) const { if (Failed) { return *this; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index fa328fe1bd..172dbcc3af 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -85,6 +85,7 @@ public: const TChecker& IsValidACL(const TString& acl, EStatus status = EStatus::StatusInvalidParameter) const; const TChecker& PQPartitionsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& PQReservedStorageLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; + const TChecker& IsExternalTable(EStatus status = EStatus::StatusNameConflict) const; }; public: diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index bb6c2e20c5..b5e8b82a80 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -777,6 +777,40 @@ void TPathDescriber::DescribeBlobDepot(const TPath& path) { Self->DescribeBlobDepot(path->PathId, path->Name, *Result->Record.MutablePathDescription()->MutableBlobDepotDescription()); } +void TPathDescriber::DescribeExternalTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl) { + const NScheme::TTypeRegistry* typeRegistry = AppData(ctx)->TypeRegistry; + auto it = Self->ExternalTables.FindPtr(pathId); + Y_VERIFY(it, "ExternalTable is not found"); + TExternalTableInfo::TPtr externalTableInfo = *it; + + auto entry = Result->Record.MutablePathDescription()->MutableExternalTableDescription(); + entry->SetName(pathEl->Name); + PathIdFromPathId(pathId, entry->MutablePathId()); + entry->SetSourceType(externalTableInfo->SourceType); + entry->SetDataSourcePath(externalTableInfo->DataSourcePath); + entry->SetLocation(externalTableInfo->Location); + entry->SetVersion(externalTableInfo->AlterVersion); + + entry->MutableColumns()->Reserve(externalTableInfo->Columns.size()); + for (auto col : externalTableInfo->Columns) { + const auto& cinfo = col.second; + if (cinfo.IsDropped()) + continue; + + auto colDescr = entry->AddColumns(); + colDescr->SetName(cinfo.Name); + colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType.GetTypeId())); // TODO: no pg type details in string type + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + colDescr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *colDescr->MutableTypeInfo() = *columnType.TypeInfo; + } + colDescr->SetId(cinfo.Id); + colDescr->SetNotNull(cinfo.NotNull); + } + entry->SetContent(externalTableInfo->Content); +} + THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe(const TActorContext& ctx) { TPathId pathId = Params.HasPathId() ? TPathId(Params.GetSchemeshardId(), Params.GetPathId()) : InvalidPathId; TString pathStr = Params.GetPath(); @@ -904,6 +938,9 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> TPathDescriber::Describe case NKikimrSchemeOp::EPathTypeBlobDepot: DescribeBlobDepot(path); break; + case NKikimrSchemeOp::EPathTypeExternalTable: + DescribeExternalTable(ctx, base->PathId, base); + break; case NKikimrSchemeOp::EPathTypeInvalid: Y_UNREACHABLE(); } diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.h b/ydb/core/tx/schemeshard/schemeshard_path_describer.h index f0d1902519..05f44ceb3f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.h @@ -43,6 +43,7 @@ class TPathDescriber { void DescribeSequence(TPathId pathId, TPathElement::TPtr pathEl); void DescribeReplication(TPathId pathId, TPathElement::TPtr pathEl); void DescribeBlobDepot(const TPath& path); + void DescribeExternalTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl); public: explicit TPathDescriber(TSchemeShard* self, NKikimrSchemeOp::TDescribePath&& params) diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp index 3deab09737..91d14a7a13 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.cpp @@ -164,6 +164,10 @@ TVirtualTimestamp TPathElement::GetDropTS() const { return TVirtualTimestamp(StepDropped, DropTxId); } +bool TPathElement::IsExternalTable() const { + return PathType == EPathType::EPathTypeExternalTable; +} + void TPathElement::SetDropped(TStepId step, TTxId txId) { PathState = EPathState::EPathStateNotExist; StepDropped = step; diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h index 0ad50bd3dd..25edaff694 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_element.h +++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h @@ -378,6 +378,7 @@ public: bool IsLikeDirectory() const; bool HasActiveChanges() const; bool IsCreateFinished() const; + bool IsExternalTable() const; TVirtualTimestamp GetCreateTS() const; TVirtualTimestamp GetDropTS() const; void SetDropped(TStepId step, TTxId txId); diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 007959c486..509f6a1418 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1619,6 +1619,19 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<PathId, AlterVersion, Description>; }; + struct ExternalTable : Table<104> { + struct OwnerPathId : Column<1, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; + struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; + struct AlterVersion : Column<3, NScheme::NTypeIds::Uint64> {}; + struct SourceType : Column<4, NScheme::NTypeIds::Utf8> {}; + struct DataSourcePath : Column<5, NScheme::NTypeIds::Utf8> {}; + struct Location : Column<6, NScheme::NTypeIds::Utf8> {}; + struct Content : Column<7, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<OwnerPathId, LocalPathId>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, SourceType, DataSourcePath, Location, AlterVersion, Content>; + }; + using TTables = SchemaTables< Paths, TxInFlight, @@ -1721,7 +1734,8 @@ struct Schema : NIceDb::Schema { Replications, ReplicationsAlterData, BlobDepots, - CdcStreamScanShardStatus + CdcStreamScanShardStatus, + ExternalTable >; static constexpr ui64 SysParam_NextPathId = 1; diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h index c150e03b85..55055400a6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h +++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h @@ -120,6 +120,9 @@ struct TTxState { item(TxAlterExtSubDomainCreateHive, 74) \ item(TxAlterCdcStreamAtTableDropSnapshot, 75) \ item(TxDropCdcStreamAtTableDropSnapshot, 76) \ + item(TxCreateExternalTable, 77) \ + item(TxDropExternalTable, 78) \ + item(TxAlterExternalTable, 79) \ // TX_STATE_TYPE_ENUM @@ -327,6 +330,7 @@ struct TTxState { case TxCreateSequence: case TxCreateReplication: case TxCreateBlobDepot: + case TxCreateExternalTable: return true; case TxInitializeBuildIndex: //this is more like alter case TxCreateCdcStreamAtTable: @@ -357,6 +361,7 @@ struct TTxState { case TxDropReplication: case TxDropBlobDepot: case TxUpdateMainTableOnIndexMove: + case TxDropExternalTable: return false; case TxAlterPQGroup: case TxAlterTable: @@ -385,6 +390,7 @@ struct TTxState { case TxAlterSequence: case TxAlterReplication: case TxAlterBlobDepot: + case TxAlterExternalTable: return false; case TxMoveTable: case TxMoveTableIndex: @@ -414,6 +420,7 @@ struct TTxState { case TxDropSequence: case TxDropReplication: case TxDropBlobDepot: + case TxDropExternalTable: return true; case TxMkDir: case TxCreateTable: @@ -445,6 +452,7 @@ struct TTxState { case TxDropCdcStreamAtTable: case TxDropCdcStreamAtTableDropSnapshot: case TxUpdateMainTableOnIndexMove: + case TxCreateExternalTable: return false; case TxAlterPQGroup: case TxAlterTable: @@ -473,6 +481,7 @@ struct TTxState { case TxAlterSequence: case TxAlterReplication: case TxAlterBlobDepot: + case TxAlterExternalTable: return false; case TxMoveTable: case TxMoveTableIndex: @@ -506,6 +515,7 @@ struct TTxState { case TxDropTableIndex: case TxRmDir: case TxFinalizeBuildIndex: + case TxDropExternalTable: return false; case TxMkDir: case TxCreateTable: @@ -535,6 +545,7 @@ struct TTxState { case TxDropCdcStreamAtTable: case TxDropCdcStreamAtTableDropSnapshot: case TxUpdateMainTableOnIndexMove: + case TxCreateExternalTable: return false; case TxAlterPQGroup: case TxAlterTable: @@ -564,6 +575,7 @@ struct TTxState { case TxAlterSequence: case TxAlterReplication: case TxAlterBlobDepot: + case TxAlterExternalTable: return false; case TxInvalid: Y_VERIFY_DEBUG("UNREACHABLE"); @@ -657,6 +669,8 @@ struct TTxState { case NKikimrSchemeOp::ESchemeOpMoveIndex: return TxInvalid; case NKikimrSchemeOp::ESchemeOpAllocatePersQueueGroup: return TxAllocatePQ; case NKikimrSchemeOp::ESchemeOpDeallocatePersQueueGroup: return TxInvalid; + case NKikimrSchemeOp::ESchemeOpCreateExternalTable: return TxCreateExternalTable; + case NKikimrSchemeOp::ESchemeOpAlterExternalTable: return TxAlterExternalTable; default: return TxInvalid; } } diff --git a/ydb/core/tx/schemeshard/ut_external_table.cpp b/ydb/core/tx/schemeshard/ut_external_table.cpp new file mode 100644 index 0000000000..e9eb58d7c4 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table.cpp @@ -0,0 +1,237 @@ +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> + +using namespace NKikimr::NSchemeShard; +using namespace NKikimr; +using namespace NKikimrSchemeOp; +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TExternalTableTest) { + Y_UNIT_TEST(CreateExternalTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateExternalTable(runtime, txId++, "/MyRoot", + R"(Name: "external_table1")", + {NKikimrScheme::StatusAccepted}); + + env.TestWaitNotification(runtime, 100); + + TestLs(runtime, "/MyRoot/external_table1", false, NLs::PathExist); + } + + Y_UNIT_TEST(DropExternalTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateExternalTable(runtime, txId++, "/MyRoot", + R"(Name: "external_table1")", + {NKikimrScheme::StatusAccepted}); + + env.TestWaitNotification(runtime, 100); + + TestLs(runtime, "/MyRoot/external_table1", false, NLs::PathExist); + + TestDropExternalTable(runtime, ++txId, "/MyRoot", "external_table1"); + env.TestWaitNotification(runtime, txId); + + TestLs(runtime, "/MyRoot/external_table1", false, NLs::PathNotExist); + } + + using TRuntimeTxFn = std::function<void(TTestBasicRuntime&, ui64)>; + + void DropTwice(const TString& path, TRuntimeTxFn createFn, TRuntimeTxFn dropFn) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + createFn(runtime, ++txId); + env.TestWaitNotification(runtime, txId); + + dropFn(runtime, ++txId); + dropFn(runtime, ++txId); + TestModificationResult(runtime, txId - 1); + + auto ev = runtime.GrabEdgeEvent<TEvSchemeShard::TEvModifySchemeTransactionResult>(); + UNIT_ASSERT(ev); + + const auto& record = ev->Record; + UNIT_ASSERT_VALUES_EQUAL(record.GetTxId(), txId); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusMultipleModifications); + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDropTxId(), txId - 1); + + env.TestWaitNotification(runtime, txId - 1); + TestDescribeResult(DescribePath(runtime, path), { + NLs::PathNotExist + }); + } + + Y_UNIT_TEST(DropTableTwice) { + auto createFn = [](TTestBasicRuntime& runtime, ui64 txId) { + TestCreateExternalTable(runtime, txId, "/MyRoot", R"( + Name: "ExternalTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + )"); + }; + + auto dropFn = [](TTestBasicRuntime& runtime, ui64 txId) { + AsyncDropExternalTable(runtime, txId, "/MyRoot", "ExternalTable"); + }; + + DropTwice("/MyRoot/ExternalTable", createFn, dropFn); + } + + Y_UNIT_TEST(ParallelCreateExternalTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 123; + + AsyncMkDir(runtime, ++txId, "/MyRoot", "DirA"); + AsyncCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "ExternalTable1" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + AsyncCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "ExternalTable2" + Columns { Name: "key1" Type: "Uint32"} + Columns { Name: "key2" Type: "Utf8"} + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + TestModificationResult(runtime, txId-2, NKikimrScheme::StatusAccepted); + TestModificationResult(runtime, txId-1, NKikimrScheme::StatusAccepted); + TestModificationResult(runtime, txId, NKikimrScheme::StatusAccepted); + + env.TestWaitNotification(runtime, {txId, txId-1, txId-2}); + + TestDescribe(runtime, "/MyRoot/DirA/ExternalTable1"); + TestDescribe(runtime, "/MyRoot/DirA/ExternalTable2"); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA"), + {NLs::PathVersionEqual(7)}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/ExternalTable1"), + {NLs::PathVersionEqual(2)}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirA/ExternalTable2"), + {NLs::PathVersionEqual(2)}); + } + + Y_UNIT_TEST(ParallelCreateSameExternalTable) { + using ESts = NKikimrScheme::EStatus; + + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 123; + + TString tableConfig = R"(Name: "NilNoviSubLuna" + Columns { Name: "key" Type: "Uint64"} + Columns { Name: "value" Type: "Uint64"})"; + + AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig); + AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig); + AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig); + + ui64 sts[3]; + sts[0] = TestModificationResults(runtime, txId-2, {ESts::StatusAccepted, ESts::StatusMultipleModifications, ESts::StatusAlreadyExists}); + sts[1] = TestModificationResults(runtime, txId-1, {ESts::StatusAccepted, ESts::StatusMultipleModifications, ESts::StatusAlreadyExists}); + sts[2] = TestModificationResults(runtime, txId, {ESts::StatusAccepted, ESts::StatusMultipleModifications, ESts::StatusAlreadyExists}); + + for (ui32 i=0; i<3; ++i) { + if (sts[i] == ESts::StatusAlreadyExists) { + TestDescribeResult(DescribePath(runtime, "/MyRoot/NilNoviSubLuna"), + {NLs::Finished, + NLs::IsExternalTable}); + } + + if (sts[i] == ESts::StatusMultipleModifications) { + TestDescribeResult(DescribePath(runtime, "/MyRoot/NilNoviSubLuna"), + {NLs::Finished, + NLs::IsExternalTable}); + } + } + + env.TestWaitNotification(runtime, {txId-2, txId-1, txId}); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/NilNoviSubLuna"), + {NLs::Finished, + NLs::IsExternalTable, + NLs::PathVersionEqual(2)}); + + TestCreateExternalTable(runtime, ++txId, "/MyRoot", tableConfig, {ESts::StatusAlreadyExists}); + + } + + + Y_UNIT_TEST(ReadOnlyMode) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 123; + + AsyncMkDir(runtime, ++txId, "/MyRoot", "SubDirA"); + AsyncCreateExternalTable(runtime, ++txId, "/MyRoot", + R"(Name: "ExternalTable1" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + // Set ReadOnly + SetSchemeshardReadOnlyMode(runtime, true); + TActorId sender = runtime.AllocateEdgeActor(); + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); + + // Verify that table creation successfully finished + env.TestWaitNotification(runtime, txId); + + // Check that describe works + TestDescribeResult(DescribePath(runtime, "/MyRoot/SubDirA"), + {NLs::Finished}); + TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable1"), + {NLs::Finished, + NLs::IsExternalTable}); + + // Check that new modifications fail + TestMkDir(runtime, ++txId, "/MyRoot", "SubDirBBBB", {NKikimrScheme::StatusReadOnly}); + TestCreateExternalTable(runtime, ++txId, "/MyRoot", + R"(Name: "ExternalTable1" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})", + {NKikimrScheme::StatusReadOnly}); + + // Disable ReadOnly + SetSchemeshardReadOnlyMode(runtime, false); + sender = runtime.AllocateEdgeActor(); + RebootTablet(runtime, TTestTxConfig::SchemeShard, sender); + + // Check that modifications now work again + TestMkDir(runtime, ++txId, "/MyRoot", "SubDirBBBB"); + } + + Y_UNIT_TEST(SchemeErrors) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 123; + + TestMkDir(runtime, ++txId, "/MyRoot", "DirA"); + env.TestWaitNotification(runtime, txId); + + TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "Table2" + Columns { Name: "RowId" Type: "BlaBlaType"})", + {NKikimrScheme::StatusSchemeError}); + TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "Table2" + Columns { Name: "" Type: "Uint64"})", + {NKikimrScheme::StatusSchemeError}); + TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "Table2" + Columns { Name: "RowId" TypeId: 27})", + {NKikimrScheme::StatusSchemeError}); + TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "Table2" + Columns { Name: "RowId" })", + {NKikimrScheme::StatusSchemeError}); + TestCreateExternalTable(runtime, ++txId, "/MyRoot/DirA", + R"(Name: "Table2" + Columns { Name: "RowId" Type: "Uint64" Id: 2} + Columns { Name: "RowId2" Type: "Uint64" Id: 2 })", + {NKikimrScheme::StatusSchemeError}); + } +} diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt new file mode 100644 index 0000000000..3b8e4769f6 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.darwin.txt @@ -0,0 +1,82 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tx-schemeshard-ut_external_table) +target_compile_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-core-tx-schemeshard-ut_external_table PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard +) +target_link_libraries(ydb-core-tx-schemeshard-ut_external_table PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-tx-schemeshard + library-cpp-getopt + cpp-regex-pcre + library-cpp-svnversion + core-testlib-default + ydb-core-tx + tx-schemeshard-ut_helpers + udf-service-exception_policy +) +target_link_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE + -Wl,-no_deduplicate + -Wl,-sdk_version,10.15 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-tx-schemeshard-ut_external_table PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table.cpp +) +set_property( + TARGET + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-core-tx-schemeshard-ut_external_table + TEST_TARGET + ydb-core-tx-schemeshard-ut_external_table + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tx-schemeshard-ut_external_table) diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..5b048e61f6 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux-aarch64.txt @@ -0,0 +1,85 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tx-schemeshard-ut_external_table) +target_compile_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-core-tx-schemeshard-ut_external_table PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard +) +target_link_libraries(ydb-core-tx-schemeshard-ut_external_table PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-lfalloc + cpp-testing-unittest_main + core-tx-schemeshard + library-cpp-getopt + cpp-regex-pcre + library-cpp-svnversion + core-testlib-default + ydb-core-tx + tx-schemeshard-ut_helpers + udf-service-exception_policy +) +target_link_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tx-schemeshard-ut_external_table PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table.cpp +) +set_property( + TARGET + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-core-tx-schemeshard-ut_external_table + TEST_TARGET + ydb-core-tx-schemeshard-ut_external_table + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tx-schemeshard-ut_external_table) diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt new file mode 100644 index 0000000000..f1418da3f2 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.linux.txt @@ -0,0 +1,87 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tx-schemeshard-ut_external_table) +target_compile_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-core-tx-schemeshard-ut_external_table PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard +) +target_link_libraries(ydb-core-tx-schemeshard-ut_external_table PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache + library-cpp-cpuid_check + cpp-testing-unittest_main + core-tx-schemeshard + library-cpp-getopt + cpp-regex-pcre + library-cpp-svnversion + core-testlib-default + ydb-core-tx + tx-schemeshard-ut_helpers + udf-service-exception_policy +) +target_link_options(ydb-core-tx-schemeshard-ut_external_table PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tx-schemeshard-ut_external_table PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table.cpp +) +set_property( + TARGET + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-core-tx-schemeshard-ut_external_table + TEST_TARGET + ydb-core-tx-schemeshard-ut_external_table + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tx-schemeshard-ut_external_table) diff --git a/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt new file mode 100644 index 0000000000..5bb4faffb4 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp b/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp new file mode 100644 index 0000000000..ea276bf960 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp @@ -0,0 +1,214 @@ +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> + +#include <ydb/core/tx/datashard/datashard.h> +#include <ydb/core/protos/flat_scheme_op.pb.h> + +#include <google/protobuf/text_format.h> + +using namespace NKikimr; +using namespace NSchemeShard; +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TExternalTableTestReboots) { + Y_UNIT_TEST(CreateExternalTableWithReboots) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + AsyncMkDir(runtime, ++t.TxId, "/MyRoot", "DirExternalTable"); + + AsyncCreateExternalTable(runtime, ++t.TxId, "/MyRoot/DirExternalTable", R"( + Name: "external_table1" + DataSourcePath: "/MySource" + Columns { Name: "a" Type: "Int32" NotNull: true } + Columns { Name: "b" Type: "Int32" NotNull: true } + )"); + + t.TestEnv->TestWaitNotification(runtime, {t.TxId, t.TxId-1}); + + { + TInactiveZone inactive(activeZone); + auto describeResult = DescribePath(runtime, "/MyRoot/DirExternalTable/external_table1"); + TestDescribeResult(describeResult, {NLs::Finished}); + + UNIT_ASSERT(describeResult.GetPathDescription().HasExternalTableDescription()); + const auto& externalTableDescription = describeResult.GetPathDescription().GetExternalTableDescription(); + UNIT_ASSERT_VALUES_EQUAL(externalTableDescription.GetName(), "external_table1"); + UNIT_ASSERT_VALUES_EQUAL(externalTableDescription.GetDataSourcePath(), "/MySource"); + UNIT_ASSERT_VALUES_EQUAL(externalTableDescription.GetVersion(), 1); + auto& columns = externalTableDescription.GetColumns(); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(columns.Get(0).GetName(), "a"); + UNIT_ASSERT_VALUES_EQUAL(columns.Get(0).GetType(), "Int32"); + UNIT_ASSERT_VALUES_EQUAL(columns.Get(0).GetNotNull(), true); + UNIT_ASSERT_VALUES_EQUAL(columns.Get(1).GetName(), "b"); + UNIT_ASSERT_VALUES_EQUAL(columns.Get(1).GetType(), "Int32"); + UNIT_ASSERT_VALUES_EQUAL(columns.Get(1).GetNotNull(), true); + } + }); + } + + Y_UNIT_TEST(ParallelCreateDrop) { //+ + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + AsyncCreateExternalTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "DropMe" + Columns { Name: "RowId" Type: "Uint64" } + Columns { Name: "Value" Type: "Utf8" } + )"); + AsyncDropExternalTable(runtime, ++t.TxId, "/MyRoot", "DropMe"); + t.TestEnv->TestWaitNotification(runtime, t.TxId-1); + + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "DropMe"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DropMe"), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(SimpleDropExternalTableWithReboots) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + "Name: \"ExternalTable\"" + "Columns { Name: \"RowId\" Type: \"Uint64\"}" + "Columns { Name: \"Value\" Type: \"Utf8\"}"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(SimpleDropExternalTableWithReboots2) { //+ + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + "Name: \"ExternalTable\"" + "Columns { Name: \"RowId\" Type: \"Uint64\"}" + "Columns { Name: \"Value\" Type: \"Utf8\"}"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(DropExternalTableWithReboots) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateExternalTable(runtime, t.TxId, "/MyRoot", + R"(Name: "ExternalTable" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"), + {NLs::PathNotExist}); + + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + R"(Name: "ExternalTable" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), + {NLs::PathNotExist}); + } + }); + } + + Y_UNIT_TEST(CreateDroppedExternalTableWithReboots) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + R"(Name: "ExternalTable" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + R"(Name: "ExternalTable" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + }); + } + + Y_UNIT_TEST(CreateDroppedExternalTableAndDropWithReboots) { //+ + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + R"(Name: "ExternalTable" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + TestCreateExternalTable(runtime, ++t.TxId, "/MyRoot", + R"(Name: "ExternalTable" + Columns { Name: "RowId" Type: "Uint64"} + Columns { Name: "Value" Type: "Utf8"})"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + TestDropExternalTable(runtime, ++t.TxId, "/MyRoot", "ExternalTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/ExternalTable"), + {NLs::PathNotExist}); + } + }); + } +} diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt new file mode 100644 index 0000000000..3af12236ae --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.darwin.txt @@ -0,0 +1,82 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tx-schemeshard-ut_external_table_reboots) +target_compile_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard +) +target_link_libraries(ydb-core-tx-schemeshard-ut_external_table_reboots PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-tx-schemeshard + library-cpp-getopt + cpp-regex-pcre + library-cpp-svnversion + core-testlib-default + ydb-core-tx + tx-schemeshard-ut_helpers + udf-service-exception_policy +) +target_link_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + -Wl,-no_deduplicate + -Wl,-sdk_version,10.15 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp +) +set_property( + TARGET + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-core-tx-schemeshard-ut_external_table_reboots + TEST_TARGET + ydb-core-tx-schemeshard-ut_external_table_reboots + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tx-schemeshard-ut_external_table_reboots) diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..e33bdbef26 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux-aarch64.txt @@ -0,0 +1,85 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tx-schemeshard-ut_external_table_reboots) +target_compile_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard +) +target_link_libraries(ydb-core-tx-schemeshard-ut_external_table_reboots PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-lfalloc + cpp-testing-unittest_main + core-tx-schemeshard + library-cpp-getopt + cpp-regex-pcre + library-cpp-svnversion + core-testlib-default + ydb-core-tx + tx-schemeshard-ut_helpers + udf-service-exception_policy +) +target_link_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp +) +set_property( + TARGET + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-core-tx-schemeshard-ut_external_table_reboots + TEST_TARGET + ydb-core-tx-schemeshard-ut_external_table_reboots + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tx-schemeshard-ut_external_table_reboots) diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt new file mode 100644 index 0000000000..50b7782671 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.linux.txt @@ -0,0 +1,87 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tx-schemeshard-ut_external_table_reboots) +target_compile_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard +) +target_link_libraries(ydb-core-tx-schemeshard-ut_external_table_reboots PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache + library-cpp-cpuid_check + cpp-testing-unittest_main + core-tx-schemeshard + library-cpp-getopt + cpp-regex-pcre + library-cpp-svnversion + core-testlib-default + ydb-core-tx + tx-schemeshard-ut_helpers + udf-service-exception_policy +) +target_link_options(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tx-schemeshard-ut_external_table_reboots PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_external_table_reboots.cpp +) +set_property( + TARGET + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-core-tx-schemeshard-ut_external_table_reboots + TEST_TARGET + ydb-core-tx-schemeshard-ut_external_table_reboots + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tx-schemeshard-ut_external_table_reboots + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tx-schemeshard-ut_external_table_reboots) diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt new file mode 100644 index 0000000000..5bb4faffb4 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_external_table_reboots/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index bcbc7ba173..4fa6ca245b 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -888,6 +888,11 @@ namespace NSchemeShardUT_Private { GENERIC_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume, &NKikimrSchemeOp::TModifyScheme::MutableDrop) DROP_BY_PATH_ID_HELPERS(DropBlockStoreVolume, NKikimrSchemeOp::EOperationType::ESchemeOpDropBlockStoreVolume) + // external table + GENERIC_HELPERS(CreateExternalTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateExternalTable, &NKikimrSchemeOp::TModifyScheme::MutableCreateExternalTable) + GENERIC_HELPERS(DropExternalTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable, &NKikimrSchemeOp::TModifyScheme::MutableDrop) + DROP_BY_PATH_ID_HELPERS(DropExternalTable, NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable) + #undef DROP_BY_PATH_ID_HELPERS #undef GENERIC_WITH_ATTRS_HELPERS #undef GENERIC_HELPERS diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index f7786d1538..49c2f892fb 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -175,6 +175,12 @@ namespace NSchemeShardUT_Private { DROP_BY_PATH_ID_HELPERS(DropTable); GENERIC_HELPERS(DropTableIndex); + + // external table + GENERIC_HELPERS(CreateExternalTable); + GENERIC_HELPERS(DropExternalTable); + DROP_BY_PATH_ID_HELPERS(DropExternalTable); + // backup & restore GENERIC_HELPERS(Backup); GENERIC_HELPERS(BackupToYt); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 975b9ae199..ebde6a638a 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -389,6 +389,13 @@ void IsTable(const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeTable); } +void IsExternalTable(const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess); + const auto& pathDescr = record.GetPathDescription(); + const auto& selfPath = pathDescr.GetSelf(); + UNIT_ASSERT_VALUES_EQUAL(selfPath.GetPathType(), NKikimrSchemeOp::EPathTypeExternalTable); +} + TCheckFunc CheckColumns(const TString& name, const TSet<TString>& columns, const TSet<TString>& droppedColumns, const TSet<TString> keyColumns, NKikimrSchemeOp::EPathState pathState) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 8043bd5fa1..0a4a266a1e 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -81,6 +81,7 @@ namespace NLs { void IsTable(const NKikimrScheme::TEvDescribeSchemeResult& record); + void IsExternalTable(const NKikimrScheme::TEvDescribeSchemeResult& record); TCheckFunc CheckColumns(const TString& name, const TSet<TString>& columns, const TSet<TString>& droppedColumns, const TSet<TString> keyColumns, NKikimrSchemeOp::EPathState pathState = NKikimrSchemeOp::EPathState::EPathStateNoChanges); void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult& record); diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp index ccd7e4ae46..ff5c90d2f5 100644 --- a/ydb/core/tx/tx_proxy/schemereq.cpp +++ b/ydb/core/tx/tx_proxy/schemereq.cpp @@ -150,6 +150,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpDropSequence: case NKikimrSchemeOp::ESchemeOpDropReplication: case NKikimrSchemeOp::ESchemeOpDropBlobDepot: + case NKikimrSchemeOp::ESchemeOpDropExternalTable: return *modifyScheme.MutableDrop()->MutableName(); case NKikimrSchemeOp::ESchemeOpAlterTable: @@ -324,6 +325,12 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpCreateBlobDepot: case NKikimrSchemeOp::ESchemeOpAlterBlobDepot: return *modifyScheme.MutableBlobDepot()->MutableName(); + + case NKikimrSchemeOp::ESchemeOpCreateExternalTable: + return *modifyScheme.MutableCreateExternalTable()->MutableName(); + + case NKikimrSchemeOp::ESchemeOpAlterExternalTable: + Y_FAIL("no implementation for ESchemeOpAlterExternalTable"); } } @@ -343,6 +350,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpCreateRtmrVolume: case NKikimrSchemeOp::ESchemeOpCreateColumnStore: case NKikimrSchemeOp::ESchemeOpCreateColumnTable: + case NKikimrSchemeOp::ESchemeOpCreateExternalTable: return true; default: return false; @@ -564,6 +572,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpAlterSequence: case NKikimrSchemeOp::ESchemeOpAlterReplication: case NKikimrSchemeOp::ESchemeOpAlterBlobDepot: + case NKikimrSchemeOp::ESchemeOpAlterExternalTable: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme))); @@ -583,7 +592,8 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpDropColumnTable: case NKikimrSchemeOp::ESchemeOpDropSequence: case NKikimrSchemeOp::ESchemeOpDropReplication: - case NKikimrSchemeOp::ESchemeOpDropBlobDepot: { + case NKikimrSchemeOp::ESchemeOpDropBlobDepot: + case NKikimrSchemeOp::ESchemeOpDropExternalTable: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = Merge(workingDir, SplitPath(GetPathNameForScheme(pbModifyScheme))); toResolve.RequiredAccess = NACLib::EAccessRights::RemoveSchema; @@ -641,6 +651,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpCreateSequence: case NKikimrSchemeOp::ESchemeOpCreateReplication: case NKikimrSchemeOp::ESchemeOpCreateBlobDepot: + case NKikimrSchemeOp::ESchemeOpCreateExternalTable: { auto toResolve = TPathToResolve(pbModifyScheme.GetOperationType()); toResolve.Path = workingDir; diff --git a/ydb/core/viewer/browse.h b/ydb/core/viewer/browse.h index 0ff4f33d94..930c5e971f 100644 --- a/ydb/core/viewer/browse.h +++ b/ydb/core/viewer/browse.h @@ -94,6 +94,8 @@ public: return NKikimrViewer::EObjectType::Replication; case NKikimrSchemeOp::EPathType::EPathTypeBlobDepot: return NKikimrViewer::EObjectType::BlobDepot; + case NKikimrSchemeOp::EPathType::EPathTypeExternalTable: + return NKikimrViewer::EObjectType::ExternalTable; case NKikimrSchemeOp::EPathType::EPathTypeExtSubDomain: case NKikimrSchemeOp::EPathType::EPathTypeTableIndex: case NKikimrSchemeOp::EPathType::EPathTypeInvalid: diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index 07e2ecf75e..8190cff37a 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -35,6 +35,7 @@ enum EObjectType { Sequence = 21; Replication = 22; BlobDepot = 23; + ExternalTable = 24; } message TBrowseInfo { diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index d5e7ee1d64..1a079fc52b 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -580,6 +580,7 @@ public: Sequence, Replication, BlobDepot, + ExternalTable }; TSchemaObject(TSchemaObject&&) = default; diff --git a/ydb/public/lib/deprecated/kicli/schema.cpp b/ydb/public/lib/deprecated/kicli/schema.cpp index 421f4de33d..a32f82d403 100644 --- a/ydb/public/lib/deprecated/kicli/schema.cpp +++ b/ydb/public/lib/deprecated/kicli/schema.cpp @@ -122,6 +122,9 @@ void TSchemaObject::Drop() { case EPathType::BlobDepot: drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropBlobDepot); break; + case EPathType::ExternalTable: + drop.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpDropExternalTable); + break; case EPathType::Unknown: case EPathType::SubDomain: case EPathType::RtmrVolume: @@ -207,6 +210,8 @@ static TSchemaObject::EPathType GetType(const NKikimrSchemeOp::TDirEntry& entry) return TSchemaObject::EPathType::Replication; case NKikimrSchemeOp::EPathTypeBlobDepot: return TSchemaObject::EPathType::BlobDepot; + case NKikimrSchemeOp::EPathTypeExternalTable: + return TSchemaObject::EPathType::ExternalTable; case NKikimrSchemeOp::EPathTypeTableIndex: case NKikimrSchemeOp::EPathTypeExtSubDomain: case NKikimrSchemeOp::EPathTypeCdcStream: diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 93f379652f..0c181e773f 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -6770,5 +6770,77 @@ "Blobs": 1 } } + }, + { + "TableId": 104, + "TableName": "ExternalTable", + "TableKey": [ + 1, + 2 + ], + "ColumnsAdded": [ + { + "ColumnId": 7, + "ColumnName": "Content", + "ColumnType": "String" + }, + { + "ColumnId": 1, + "ColumnName": "OwnerPathId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 2, + "ColumnName": "LocalPathId", + "ColumnType": "Uint64" + }, + { + "ColumnId": 3, + "ColumnName": "AlterVersion", + "ColumnType": "Uint64" + }, + { + "ColumnId": 4, + "ColumnName": "SourceType", + "ColumnType": "Utf8" + }, + { + "ColumnId": 5, + "ColumnName": "DataSourcePath", + "ColumnType": "Utf8" + }, + { + "ColumnId": 6, + "ColumnName": "Location", + "ColumnType": "Utf8" + } + ], + "ColumnsDropped": [], + "ColumnFamilies": { + "0": { + "Columns": [ + 7, + 1, + 2, + 3, + 4, + 5, + 6 + ], + "RoomID": 0, + "Codec": 0, + "InMemory": false, + "Cache": 0, + "Small": 4294967295, + "Large": 4294967295 + } + }, + "Rooms": { + "0": { + "Main": 1, + "Outer": 1, + "Blobs": 1 + } + } } ]
\ No newline at end of file |