diff options
author | ilnaz <ilnaz@ydb.tech> | 2022-10-05 12:50:08 +0300 |
---|---|---|
committer | ilnaz <ilnaz@ydb.tech> | 2022-10-05 12:50:08 +0300 |
commit | 6935fc72381ab994d0b2e6bd4fbeb2def684828e (patch) | |
tree | ae45a722ed702607faa06d673636fc838ff69cc8 | |
parent | 22a9f2946efed8beb7d3c68dbeb8c14d5f63edf9 (diff) | |
download | ydb-6935fc72381ab994d0b2e6bd4fbeb2def684828e.tar.gz |
Common CreateLock
18 files changed, 316 insertions, 372 deletions
diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto index 0c7f4db41c3..874bf73b806 100644 --- a/ydb/core/protos/counters_schemeshard.proto +++ b/ydb/core/protos/counters_schemeshard.proto @@ -79,7 +79,7 @@ enum ESimpleCounters { COUNTER_IN_FLIGHT_OPS_TxInitializeBuildIndex = 65 [(CounterOpts) = {Name: "InFlightOps/InitializeBuildIndex"}]; COUNTER_SNAPSHOTS_COUNT = 66 [(CounterOpts) = {Name: "Snapshots"}]; COUNTER_LOCKS_COUNT = 67 [(CounterOpts) = {Name: "Locks"}]; - COUNTER_IN_FLIGHT_OPS_TxCreateLockForIndexBuild = 68 [(CounterOpts) = {Name: "InFlightOps/CreateLockForIndexBuild"}]; + COUNTER_IN_FLIGHT_OPS_TxCreateLock = 68 [(CounterOpts) = {Name: "InFlightOps/CreateLock"}]; COUNTER_IN_FLIGHT_OPS_TxAlterTableIndex = 69 [(CounterOpts) = {Name: "InFlightOps/AlterTableIndex"}]; COUNTER_IN_FLIGHT_OPS_TxFinalizeBuildIndex = 70 [(CounterOpts) = {Name: "InFlightOps/FinalizeBuildIndex"}]; COUNTER_IN_FLIGHT_OPS_TxAlterSolomonVolume = 71 [(CounterOpts) = {Name: "InFlightOps/AlterSolomonVolume"}]; @@ -212,7 +212,7 @@ enum ECumulativeCounters { COUNTER_FINISHED_OPS_TxUpgradeSubDomain = 37 [(CounterOpts) = {Name: "FinishedOps/UpgradeSubDomain"}]; COUNTER_FINISHED_OPS_TxUpgradeSubDomainDecision = 38 [(CounterOpts) = {Name: "FinishedOps/UpgradeSubDomainDecision"}]; COUNTER_FINISHED_OPS_TxInitializeBuildIndex = 39 [(CounterOpts) = {Name: "FinishedOps/InitializeBuildIndex"}]; - COUNTER_FINISHED_OPS_TxCreateLockForIndexBuild = 40 [(CounterOpts) = {Name: "FinishedOps/CreateLockForIndexBuild"}]; + COUNTER_FINISHED_OPS_TxCreateLock = 40 [(CounterOpts) = {Name: "FinishedOps/CreateLock"}]; COUNTER_FINISHED_OPS_TxAlterTableIndex = 41 [(CounterOpts) = {Name: "FinishedOps/AlterTableIndex"}]; COUNTER_FINISHED_OPS_TxFinalizeBuildIndex = 42 [(CounterOpts) = {Name: "FinishedOps/FinalizeBuildIndex"}]; COUNTER_FINISHED_OPS_TxAlterSolomonVolume = 43 [(CounterOpts) = {Name: "FinishedOps/AlterSolomonVolume"}]; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 7f4d35655c5..3281245d02a 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1176,7 +1176,7 @@ enum EOperationType { ESchemeOpUpgradeSubDomainDecision = 37; ESchemeOpCreateIndexBuild = 38; ESchemeOpInitiateBuildIndexMainTable = 39; - ESchemeOpCreateLockForIndexBuild = 40; + ESchemeOpCreateLock = 40; ESchemeOpApplyIndexBuild = 41; ESchemeOpFinalizeBuildIndexMainTable = 42; ESchemeOpAlterTableIndex = 43; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp index 3d727f4516b..dab91269b34 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp @@ -893,8 +893,8 @@ ISubOperationBase::TPtr TOperation::RestorePart(TTxState::ETxType txType, TTxSta return CreateDropTableIndexAtMainTable(NextPartId(), txState); case TTxState::ETxType::TxUpdateMainTableOnIndexMove: return CreateUpdateMainTableOnIndexMove(NextPartId(), txState); - case TTxState::ETxType::TxCreateLockForIndexBuild: - return CreateLockForIndexBuild(NextPartId(), txState); + case TTxState::ETxType::TxCreateLock: + return CreateLock(NextPartId(), txState); case TTxState::ETxType::TxDropLock: return DropLock(NextPartId(), txState); case TTxState::ETxType::TxAlterTableIndex: @@ -1057,8 +1057,8 @@ ISubOperationBase::TPtr TOperation::ConstructPart(NKikimrSchemeOp::EOperationTyp return CreateUpgradeSubDomainDecision(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexBuild: Y_FAIL("multipart operations are handled before, also they require transaction details"); - case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLockForIndexBuild: - return CreateLockForIndexBuild(NextPartId(), tx); + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLock: + return CreateLock(NextPartId(), tx); case NKikimrSchemeOp::EOperationType::ESchemeOpDropLock: return DropLock(NextPartId(), tx); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp index 417fc988061..ab13a1c0d1a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp @@ -1,85 +1,109 @@ #include "schemeshard__operation_part.h" #include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" #include "schemeshard_path_element.h" #include "schemeshard_utils.h" -#include "schemeshard_impl.h" - -#include <ydb/core/base/table_index.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/protos/flat_scheme_op.pb.h> -#include <ydb/library/yql/minikql/mkql_type_ops.h> - -namespace NKikimr { -namespace NSchemeShard { +namespace NKikimr::NSchemeShard { -TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { +TVector<ISubOperationBase::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexBuild); - auto config = tx.GetInitiateIndexBuild(); - TString tablePath = config.GetTable(); - const NKikimrSchemeOp::TIndexCreationConfig& indexConfig = config.GetIndex(); + const auto& op = tx.GetInitiateIndexBuild(); + const auto& indexDesc = op.GetIndex(); - TPath table = TPath::Resolve(tablePath, context.SS); - TPath index = table.Child(indexConfig.GetName()); + const auto table = TPath::Resolve(op.GetTable(), context.SS); + const auto index = table.Child(indexDesc.GetName()); + { + const auto checks = index.Check(); + checks + .IsAtLocalSchemeShard(); + + if (index.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() + .FailOnExist(TPathElement::EPathType::EPathTypeTableIndex, false); + } else { + checks + .NotEmpty() + .NotResolved(); + } + checks + .IsValidLeafName() + .PathsLimit(2) // index and impl-table + .DirChildrenLimit() + .ShardsLimit(1); // impl-table + + if (!checks) { + TString explain = TStringBuilder() << "path checks failed, path: " << index.PathString(); + const auto status = checks.GetStatus(&explain); + return {CreateReject(opId, status, explain)}; + } + } - TTableInfo::TPtr tableInfo = context.SS->Tables.at(table.Base()->PathId); + auto tableInfo = context.SS->Tables.at(table.Base()->PathId); + auto domainInfo = table.DomainInfo(); - //check idempotence + const ui64 aliveIndices = context.SS->GetAliveChildren(table.Base(), NKikimrSchemeOp::EPathTypeTableIndex); + if (aliveIndices + 1 > domainInfo->GetSchemeLimits().MaxTableIndices) { + return {CreateReject(opId, NKikimrScheme::EStatus::StatusPreconditionFailed, TStringBuilder() + << "indexes count has reached maximum value in the table" + << ", children limit for dir in domain: " << domainInfo->GetSchemeLimits().MaxTableIndices + << ", intention to create new children: " << aliveIndices + 1)}; + } - //check limits + NTableIndex::TTableColumns implTableColumns; + NKikimrScheme::EStatus status; + TString errStr; + if (!NTableIndex::CommonCheck(tableInfo, indexDesc, domainInfo->GetSchemeLimits(), false, implTableColumns, status, errStr)) { + return {CreateReject(opId, status, errStr)}; + } TVector<ISubOperationBase::TPtr> result; { - auto tableIndexCreation = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTableIndex); - *tableIndexCreation.MutableLockGuard() = tx.GetLockGuard(); - tableIndexCreation.MutableCreateTableIndex()->CopyFrom(indexConfig); + auto outTx = TransactionTemplate(table.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTableIndex); + *outTx.MutableLockGuard() = tx.GetLockGuard(); + outTx.MutableCreateTableIndex()->CopyFrom(indexDesc); + outTx.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly); - if (!indexConfig.HasType()) { - tableIndexCreation.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal); + if (!indexDesc.HasType()) { + outTx.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal); } - tableIndexCreation.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly); - result.push_back(CreateNewTableIndex(NextPartId(nextId, result), tableIndexCreation)); + result.push_back(CreateNewTableIndex(NextPartId(opId, result), outTx)); } { - auto initialize = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable); - *initialize.MutableLockGuard() = tx.GetLockGuard(); - auto& shapshot = *initialize.MutableInitiateBuildIndexMainTable(); + auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable); + *outTx.MutableLockGuard() = tx.GetLockGuard(); + + auto& shapshot = *outTx.MutableInitiateBuildIndexMainTable(); shapshot.SetTableName(table.LeafName()); - result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(nextId, result), initialize)); + result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(opId, result), outTx)); } { - auto indexImplTableCreation = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable); - NTableIndex::TTableColumns baseTableColumns = NTableIndex::ExtractInfo(tableInfo); - NTableIndex::TIndexColumns indexKeys = NTableIndex::ExtractInfo(indexConfig); + auto outTx = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable); + auto& indexImplTableDescription = *outTx.MutableCreateTable(); - TString explain; - if (!NTableIndex::IsCompatibleIndex(baseTableColumns, indexKeys, explain)) { - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, explain)}; - } - - NTableIndex::TTableColumns impTableColumns = NTableIndex::CalcTableImplDescription(baseTableColumns, indexKeys); - - auto& indexImplTableDescription = *indexImplTableCreation.MutableCreateTable(); // This description provided by user to override partition policy - const auto& userIndexDesc = indexConfig.GetIndexImplTableDescription(); - indexImplTableDescription = NTableIndex::CalcImplTableDesc(tableInfo, impTableColumns, userIndexDesc); + const auto& userIndexDesc = indexDesc.GetIndexImplTableDescription(); + indexImplTableDescription = CalcImplTableDesc(tableInfo, implTableColumns, userIndexDesc); indexImplTableDescription.MutablePartitionConfig()->MutableCompactionPolicy()->SetKeepEraseMarkers(true); indexImplTableDescription.MutablePartitionConfig()->SetShadowData(true); - result.push_back(CreateInitializeBuildIndexImplTable(NextPartId(nextId, result), indexImplTableCreation)); + result.push_back(CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx)); } return result; } } -} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index ce8b641418a..6fde4d7cbaf 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -1,25 +1,16 @@ #include "schemeshard__operation_part.h" #include "schemeshard__operation_common.h" +#include "schemeshard_impl.h" #include "schemeshard_path_element.h" #include "schemeshard_utils.h" -#include "schemeshard_impl.h" - -#include <ydb/core/base/table_index.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/protos/flat_scheme_op.pb.h> -namespace { +namespace NKikimr::NSchemeShard { -using namespace NKikimr; -using namespace NSchemeShard; using namespace NTableIndex; -} - -namespace NKikimr { -namespace NSchemeShard { - TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable); @@ -35,6 +26,7 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T indexedTableShards += 1; } } + ui32 sequencesCount = indexedTable.SequenceDescriptionSize(); ui32 baseShards = TTableInfo::ShardsToCreate(baseTableDescription); ui32 shardsToCreate = baseShards + indexedTableShards; @@ -117,46 +109,14 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; } - TIndexColumns indexKeys = ExtractInfo(indexDescription); - if (indexKeys.KeyColumns.empty()) { - TString msg = TStringBuilder() << "no key colums in index creation config"; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; - } - - if (!indexKeys.DataColumns.empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { - TString msg = TStringBuilder() << "It is not allowed to create index with data column"; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; - } - - TString explainErr; - if (!IsCompatibleIndex(baseTableColumns, indexKeys, explainErr)) { - TString msg = TStringBuilder() << "IsCompatibleIndex fail with explain: " << explainErr; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; - } - - TTableColumns impTableColumns = CalcTableImplDescription(baseTableColumns, indexKeys); - - TColumnTypes columnsTypes; - if (!ExtractTypes(baseTableDescription, columnsTypes, explainErr)) { - TString msg = TStringBuilder() << "ExtractTypes fail with explain: " << explainErr; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; - } - - if (!IsCompatibleKeyTypes(columnsTypes, impTableColumns, uniformIndexTable, explainErr)) { - TString msg = TStringBuilder() << "IsCompatibleKeyTypes fail with explain: " << explainErr; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)}; - } - - if (impTableColumns.Keys.size() > domainInfo->GetSchemeLimits().MaxTableKeyColumns) { - TString msg = TStringBuilder() - << "Too many key indexed, index table reaches the limit of the maximum keys colums count" - << ": indexing colums: " << indexKeys.KeyColumns.size() - << ": requested keys colums for index table: " << impTableColumns.Keys.size() - << ". Limit: " << domainInfo->GetSchemeLimits().MaxTableKeyColumns; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusSchemeError, msg)}; + TTableColumns implTableColumns; + NKikimrScheme::EStatus status; + TString errStr; + if (!CommonCheck(baseTableDescription, indexDescription, domainInfo->GetSchemeLimits(), uniformIndexTable, implTableColumns, status, errStr)) { + return {CreateReject(nextId, status, errStr)}; } - indexes.emplace(indexName, std::move(impTableColumns)); + indexes.emplace(indexName, std::move(implTableColumns)); } THashSet<TString> sequences; @@ -265,12 +225,12 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); scheme.SetFailOnExist(tx.GetFailOnExist()); - TTableColumns impTableColumns = indexes.at(indexDescription.GetName()); + const auto& implTableColumns = indexes.at(indexDescription.GetName()); auto& indexImplTableDescription = *scheme.MutableCreateTable(); // This description provided by user to override partition policy const auto& userIndexDesc = indexDescription.GetIndexImplTableDescription(); - indexImplTableDescription = CalcImplTableDesc(baseTableDescription, impTableColumns, userIndexDesc); + indexImplTableDescription = CalcImplTableDesc(baseTableDescription, implTableColumns, userIndexDesc); result.push_back(CreateNewTable(NextPartId(nextId, result), scheme)); } @@ -291,4 +251,3 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T } } -} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index af5a2473a5b..7e16c0812b7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -1,39 +1,36 @@ #include "schemeshard__operation_part.h" #include "schemeshard__operation_common.h" #include "schemeshard_impl.h" -#include "schemeshard_utils.h" - -#include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/base/subdomain.h> +#include <ydb/core/protos/flat_scheme_op.pb.h> -namespace { +#define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) -using namespace NKikimr; -using namespace NSchemeShard; +namespace NKikimr::NSchemeShard { +namespace { -class TCreateLockForIndexBuild: public TSubOperation { +class TCreateLock: public TSubOperation { const TOperationId OperationId; const TTxTransaction Transaction; - TTxState::ETxState State = TTxState::Invalid; + TTxState::ETxState State; TTxState::ETxState NextState() { return TTxState::Done; } TTxState::ETxState NextState(TTxState::ETxState state) { - switch(state) { + switch (state) { case TTxState::Waiting: return TTxState::Done; default: return TTxState::Invalid; } - return TTxState::Invalid; } TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) { - switch(state) { + switch (state) { case TTxState::Waiting: case TTxState::Done: return THolder(new TDone(OperationId)); @@ -52,47 +49,38 @@ class TCreateLockForIndexBuild: public TSubOperation { } public: - TCreateLockForIndexBuild(TOperationId id, const TTxTransaction& tx) + explicit TCreateLock(TOperationId id, const TTxTransaction& tx) : OperationId(id) - , Transaction(tx) + , Transaction(tx) + , State(TTxState::Invalid) { } - TCreateLockForIndexBuild(TOperationId id, TTxState::ETxState state) + explicit TCreateLock(TOperationId id, TTxState::ETxState state) : OperationId(id) - , State(state) + , State(state) { SetState(SelectStateFunc(state)); } THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override { - const TTabletId ssId = context.SS->SelfTabletId(); - - auto lockSchema = Transaction.GetLockConfig(); + const auto& workingDir = Transaction.GetWorkingDir(); + const auto& op = Transaction.GetLockConfig(); - const TString& parentPathStr = Transaction.GetWorkingDir(); + LOG_N("TCreateLock Propose" + << ": opId# " << OperationId + << ", path# " << workingDir << "/" << op.GetName()); - LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TCreateLockForIndexBuild Propose" - << ", path: " << parentPathStr << "/" << lockSchema.GetName() - << ", opId: " << OperationId - << ", at schemeshard: " << ssId); - - auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId)); + auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), context.SS->TabletID()); if (!Transaction.HasLockConfig()) { result->SetError(NKikimrScheme::StatusInvalidParameter, "no locking config present"); return result; } - if (!Transaction.HasInitiateIndexBuild()) { - result->SetError(NKikimrScheme::StatusInvalidParameter, "no build index config present"); - return result; - } - - NSchemeShard::TPath parentPath = NSchemeShard::TPath::Resolve(parentPathStr, context.SS); + const auto parentPath = TPath::Resolve(workingDir, context.SS); { - NSchemeShard::TPath::TChecker checks = parentPath.Check(); + const auto checks = parentPath.Check(); checks .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() @@ -103,17 +91,16 @@ public: .IsLikeDirectory(); if (!checks) { - TString explain = TStringBuilder() << "parent path fail checks" - << ", path: " << parentPath.PathString(); - auto status = checks.GetStatus(&explain); + TString explain = TStringBuilder() << "path checks failed, path: " << parentPath.PathString(); + const auto status = checks.GetStatus(&explain); result->SetError(status, explain); return result; } } - NSchemeShard::TPath tablePath = parentPath.Child(lockSchema.GetName()); + const auto tablePath = parentPath.Child(op.GetName()); { - NSchemeShard::TPath::TChecker checks = tablePath.Check(); + const auto checks = tablePath.Check(); checks .IsAtLocalSchemeShard() .IsResolved() @@ -124,9 +111,8 @@ public: .IsCommonSensePath(); if (!checks) { - TString explain = TStringBuilder() << "table path fail checks" - << ", path: " << tablePath.PathString(); - auto status = checks.GetStatus(&explain); + TString explain = TStringBuilder() << "path checks failed, path: " << tablePath.PathString(); + const auto status = checks.GetStatus(&explain); result->SetError(status, explain); if (tablePath.IsResolved()) { result->SetPathCreateTxId(ui64(tablePath.Base()->CreateTxId)); @@ -136,108 +122,13 @@ public: } } - auto buildIndexSchema = Transaction.GetInitiateIndexBuild(); - - if (buildIndexSchema.GetTable() != tablePath.PathString()) { - result->SetError(NKikimrScheme::StatusInvalidParameter, "table path in build index mismatch with locking path"); - return result; - } - - auto indexSchema = buildIndexSchema.GetIndex(); - - NSchemeShard::TPath indexPath = tablePath.Child(indexSchema.GetName()); - { - NSchemeShard::TPath::TChecker checks = indexPath.Check(); - checks - .IsAtLocalSchemeShard(); - - if (indexPath.IsResolved()) { - checks - .IsResolved() - .NotUnderDeleting() - .FailOnExist(TPathElement::EPathType::EPathTypeTableIndex, false); - } else { - checks - .NotEmpty() - .NotResolved(); - } - - checks - .IsValidLeafName() - .PathsLimit(2) // we are creating 2 pathes at next stages: index and table - .DirChildrenLimit() - .ShardsLimit(1) // we are creating 1 shard at next stages for index table - ; - - if (!checks) { - TString explain = TStringBuilder() << "index path fail checks" - << ", path: " << indexPath.PathString(); - auto status = checks.GetStatus(&explain); - result->SetError(status, explain); - if (indexPath.IsResolved()) { - result->SetPathCreateTxId(ui64(indexPath.Base()->CreateTxId)); - result->SetPathId(indexPath.Base()->PathId.LocalPathId); - } - return result; - } - } - - TPathElement::TPtr pathEl = tablePath.Base(); - TPathId pathId = pathEl->PathId; + const auto pathId = tablePath.Base()->PathId; result->SetPathId(pathId.LocalPathId); - TTableInfo::TPtr tableInfo = context.SS->Tables.at(tablePath.Base()->PathId); - - const ui64 aliveIndices = context.SS->GetAliveChildren(tablePath.Base(), NKikimrSchemeOp::EPathTypeTableIndex); - if (aliveIndices + 1 > tablePath.DomainInfo()->GetSchemeLimits().MaxTableIndices) { - auto msg = TStringBuilder() << "indexes count has reached maximum value in the table" - << ", children limit for dir in domain: " << tablePath.DomainInfo()->GetSchemeLimits().MaxTableIndices - << ", intention to create new children: " << aliveIndices + 1; - result->SetError(NKikimrScheme::StatusPreconditionFailed, msg); - return result; - } - - { - NTableIndex::TIndexColumns indexKeys = NTableIndex::ExtractInfo(indexSchema); - if (indexKeys.KeyColumns.empty()) { - result->SetError(NKikimrScheme::StatusInvalidParameter, "no key colums in index creation config"); - return result; - } - - NTableIndex::TTableColumns baseTableColumns = NTableIndex::ExtractInfo(tableInfo); - - TString explainErr; - if (!NTableIndex::IsCompatibleIndex(baseTableColumns, indexKeys, explainErr)) { - TString msg = TStringBuilder() << "IsCompatibleIndex fail with explain: " << explainErr; - result->SetError(NKikimrScheme::StatusInvalidParameter, msg); - return result; - } - - NTableIndex::TTableColumns impTableColumns = NTableIndex::CalcTableImplDescription(baseTableColumns, indexKeys); - - if (!NTableIndex::IsCompatibleKeyTypes(NTableIndex::ExtractTypes(tableInfo), impTableColumns, false, explainErr)) { - TString msg = TStringBuilder() << "IsCompatibleKeyTypes fail with explain: " << explainErr; - result->SetError(NKikimrScheme::StatusInvalidParameter, msg); - return result; - } - - if (impTableColumns.Keys.size() > tablePath.DomainInfo()->GetSchemeLimits().MaxTableKeyColumns) { - TString msg = TStringBuilder() - << "Too many key indexed, index table reaches the limit of the maximum keys colums count" - << ": indexing colums: " << indexKeys.KeyColumns.size() - << ": requested keys colums for index table: " << impTableColumns.Keys.size() - << ". Limit: " << tablePath.DomainInfo()->GetSchemeLimits().MaxTableKeyColumns; - result->SetError(NKikimrScheme::StatusSchemeError, msg); - return result; - } - } - if (tablePath.LockedBy() == OperationId.GetTxId()) { - TString explain = TStringBuilder() - << "dst path fail checks" + result->SetError(TEvSchemeShard::EStatus::StatusAlreadyExists, TStringBuilder() << "path checks failed" << ", path already locked by this operation" - << ", path: " << tablePath.PathString(); - result->SetError(TEvSchemeShard::EStatus::StatusAlreadyExists, explain); + << ", path: " << tablePath.PathString()); return result; } @@ -246,27 +137,24 @@ public: result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); return result; } - if (!context.SS->CheckInFlightLimit(TTxState::TxCreateLockForIndexBuild, errStr)) { + + if (!context.SS->CheckInFlightLimit(TTxState::TxCreateLock, errStr)) { result->SetError(NKikimrScheme::StatusResourceExhausted, errStr); return result; } - NIceDb::TNiceDb db(context.GetDB()); - - pathEl->LastTxId = OperationId.GetTxId(); - pathEl->PathState = NKikimrSchemeOp::EPathState::EPathStateAlter; + tablePath.Base()->LastTxId = OperationId.GetTxId(); + tablePath.Base()->PathState = NKikimrSchemeOp::EPathState::EPathStateAlter; - TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateLockForIndexBuild, pathId); + TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateLock, pathId); txState.State = TTxState::Done; + NIceDb::TNiceDb db(context.GetDB()); context.SS->PersistTxState(db, OperationId); - if (pathEl->IsTable()) { - // wait all the splits - TTableInfo::TPtr table = context.SS->Tables.at(pathId); - for (auto splitTx: table->GetSplitOpsInFlight()) { - context.OnComplete.Dependence(splitTx.GetTxId(), OperationId.GetTxId()); - } + auto table = context.SS->Tables.at(pathId); + for (const auto& splitTx : table->GetSplitOpsInFlight()) { + context.OnComplete.Dependence(splitTx.GetTxId(), OperationId.GetTxId()); } context.SS->LockedPaths[pathId] = OperationId.GetTxId(); @@ -277,6 +165,7 @@ public: State = NextState(); SetState(SelectStateFunc(State)); + return result; } @@ -285,29 +174,22 @@ public: } void AbortUnsafe(TTxId forceDropTxId, TOperationContext& context) override { - LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TCreateLock AbortUnsafe" - << ", opId: " << OperationId - << ", forceDropId: " << forceDropTxId - << ", at schemeshard: " << context.SS->TabletID()); - + LOG_N("TCreateLock AbortUnsafe" + << ": opId# " << OperationId + << ", txId# " << forceDropTxId); context.OnComplete.DoneOperation(OperationId); } }; -} - -namespace NKikimr { -namespace NSchemeShard { +} // anonymous namespace -ISubOperationBase::TPtr CreateLockForIndexBuild(TOperationId id, const TTxTransaction& tx) { - return new TCreateLockForIndexBuild(id, tx); +ISubOperationBase::TPtr CreateLock(TOperationId id, const TTxTransaction& tx) { + return new TCreateLock(id, tx); } -ISubOperationBase::TPtr CreateLockForIndexBuild(TOperationId id, TTxState::ETxState state) { +ISubOperationBase::TPtr CreateLock(TOperationId id, TTxState::ETxState state) { Y_VERIFY(state != TTxState::Invalid); - return new TCreateLockForIndexBuild(id, state); + return new TCreateLock(id, state); } } -} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index 8d42df68937..469cc602845 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -443,8 +443,8 @@ ISubOperationBase::TPtr CreateFinalizeBuildIndexImplTable(TOperationId id, TTxSt ISubOperationBase::TPtr CreateFinalizeBuildIndexMainTable(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr CreateFinalizeBuildIndexMainTable(TOperationId id, TTxState::ETxState state); -ISubOperationBase::TPtr CreateLockForIndexBuild(TOperationId id, const TTxTransaction& tx); -ISubOperationBase::TPtr CreateLockForIndexBuild(TOperationId id, TTxState::ETxState state); +ISubOperationBase::TPtr CreateLock(TOperationId id, const TTxTransaction& tx); +ISubOperationBase::TPtr CreateLock(TOperationId id, TTxState::ETxState state); ISubOperationBase::TPtr DropLock(TOperationId id, const TTxTransaction& tx); ISubOperationBase::TPtr DropLock(TOperationId id, TTxState::ETxState state); diff --git a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp index 289e80451df..e3bd38fe627 100644 --- a/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_audit_log_fragment.cpp @@ -86,7 +86,7 @@ TString DefineUserOperationName(NKikimrSchemeOp::EOperationType type) { return "BUILD INDEX"; case NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable: return "ALTER TABLE BUILD INDEX INIT"; - case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLockForIndexBuild: + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLock: return "ALTER TABLE LOCK"; case NKikimrSchemeOp::EOperationType::ESchemeOpApplyIndexBuild: return "ALTER TABLE BUILD INDEX APPLY"; @@ -329,7 +329,7 @@ void TAuditLogFragment::FillPathes(const NKikimrSchemeOp::TModifyScheme& tx) { case NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable: Path = JoinPath({tx.GetWorkingDir(), tx.GetInitiateBuildIndexMainTable().GetTableName()}); break; - case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLockForIndexBuild: + case NKikimrSchemeOp::EOperationType::ESchemeOpCreateLock: Path = JoinPath({tx.GetWorkingDir(), tx.GetLockConfig().GetName()}); break; case NKikimrSchemeOp::EOperationType::ESchemeOpApplyIndexBuild: diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp index d1de0a0cb94..ac5a7c600df 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp @@ -1,19 +1,17 @@ #include "schemeshard_build_index.h" -#include "schemeshard_impl.h" #include "schemeshard_build_index_helpers.h" #include "schemeshard_build_index_tx_base.h" +#include "schemeshard_impl.h" +#include "schemeshard_utils.h" #include <ydb/public/api/protos/ydb_issue_message.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -namespace NKikimr { -namespace NSchemeShard { +namespace NKikimr::NSchemeShard { using namespace NTabletFlatExecutor; class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder::TTxBase { -private: TEvIndexBuilder::TEvCreateRequest::TPtr Request; public: @@ -28,7 +26,8 @@ public: } bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override { - const NKikimrIndexBuilder::TEvCreateRequest& request = Request->Get()->Record; + const auto& request = Request->Get()->Record; + const auto& settings = request.GetSettings(); LOG_N("TIndexBuilder::TTxCreate: DoExecute" << ", Database: " << request.GetDatabaseName() @@ -39,37 +38,21 @@ public: auto response = MakeHolder<TEvIndexBuilder::TEvCreateResponse>(request.GetTxId()); - const auto& dataColumns = request.GetSettings().index().data_columns(); - - if (!dataColumns.empty() && !Self->AllowDataColumnForIndexTable) { - return Reply( - std::move(response), - Ydb::StatusIds::UNSUPPORTED, - TStringBuilder() << "Creating covered index is unsupported yet" - ); - } - const auto id = TIndexBuildId(request.GetTxId()); if (Self->IndexBuilds.contains(id)) { - return Reply( - std::move(response), - Ydb::StatusIds::ALREADY_EXISTS, - TStringBuilder() << "Index build with id '" << id << "' already exists" - ); + return Reply(std::move(response), Ydb::StatusIds::ALREADY_EXISTS, TStringBuilder() + << "Index build with id '" << id << "' already exists"); } const TString& uid = GetUid(request.GetOperationParams().labels()); if (uid && Self->IndexBuildsByUid.contains(uid)) { - return Reply( - std::move(response), - Ydb::StatusIds::ALREADY_EXISTS, - TStringBuilder() << "Index build with uid '" << uid << "' already exists" - ); + return Reply(std::move(response), Ydb::StatusIds::ALREADY_EXISTS, TStringBuilder() + << "Index build with uid '" << uid << "' already exists"); } - const TPath domainPath = TPath::Resolve(request.GetDatabaseName(), Self); + const auto domainPath = TPath::Resolve(request.GetDatabaseName(), Self); { - TPath::TChecker checks = domainPath.Check(); + const auto checks = domainPath.Check(); checks .IsAtLocalSchemeShard() .IsResolved() @@ -80,37 +63,28 @@ public: if (!checks) { TString explain; - auto status = checks.GetStatus(&explain); - - return Reply( - std::move(response), - TranslateStatusCode(status), - TStringBuilder() << "Failed database check: " << explain - ); + const auto status = checks.GetStatus(&explain); + return Reply(std::move(response), TranslateStatusCode(status), TStringBuilder() + << "Failed database check: " << explain); } } - NIceDb::TNiceDb db(txc.DB); - auto subDomainPathId = domainPath.GetPathIdForDomain(); auto subDomainInfo = domainPath.DomainInfo(); - bool quotaAcquired = subDomainInfo->TryConsumeSchemeQuota(ctx.Now()); + const bool quotaAcquired = subDomainInfo->TryConsumeSchemeQuota(ctx.Now()); + NIceDb::TNiceDb db(txc.DB); // We need to persist updated/consumed quotas even if operation fails for other reasons Self->PersistSubDomainSchemeQuotas(db, subDomainPathId, *subDomainInfo); if (!quotaAcquired) { - return Reply( - std::move(response), - Ydb::StatusIds::OVERLOADED, + return Reply(std::move(response), Ydb::StatusIds::OVERLOADED, "Request exceeded a limit on the number of schema operations, try again later."); } - const auto& settings = request.GetSettings(); - - const TPath path = TPath::Resolve(settings.source_path(), Self); + const auto tablePath = TPath::Resolve(settings.source_path(), Self); { - TPath::TChecker checks = path.Check(); + const auto checks = tablePath.Check(); checks .IsAtLocalSchemeShard() .IsResolved() @@ -122,25 +96,65 @@ public: if (!checks) { TString explain; - auto status = checks.GetStatus(&explain); + const auto status = checks.GetStatus(&explain); + return Reply(std::move(response), TranslateStatusCode(status), TStringBuilder() + << "Failed table check: " << explain); + } + } - return Reply( - std::move(response), - TranslateStatusCode(status), - TStringBuilder() << "Failed table check: " << explain - ); + const auto indexPath = tablePath.Child(settings.index().name()); + { + const auto checks = indexPath.Check(); + checks + .IsAtLocalSchemeShard(); + + if (indexPath.IsResolved()) { + checks + .IsResolved() + .NotUnderDeleting() + .FailOnExist(TPathElement::EPathType::EPathTypeTableIndex, false); + } else { + checks + .NotEmpty() + .NotResolved(); } + + checks + .IsValidLeafName() + .PathsLimit(2) // index and impl-table + .DirChildrenLimit() + .ShardsLimit(1); // impl-table + + if (!checks) { + TString explain; + const auto status = checks.GetStatus(&explain); + return Reply(std::move(response), TranslateStatusCode(status), TStringBuilder() + << "Failed index check: " << explain); + } + } + + auto tableInfo = Self->Tables.at(tablePath.Base()->PathId); + auto domainInfo = tablePath.DomainInfo(); + + const ui64 aliveIndices = Self->GetAliveChildren(tablePath.Base(), NKikimrSchemeOp::EPathTypeTableIndex); + if (aliveIndices + 1 > domainInfo->GetSchemeLimits().MaxTableIndices) { + return Reply(std::move(response), Ydb::StatusIds::PRECONDITION_FAILED, TStringBuilder() + << "indexes count has reached maximum value in the table" + << ", children limit for dir in domain: " << domainInfo->GetSchemeLimits().MaxTableIndices + << ", intention to create new children: " << aliveIndices + 1); } TIndexBuildInfo::TPtr buildInfo = new TIndexBuildInfo(id, uid); TString explain; - if (!Prepare(buildInfo, domainPath, path, settings, explain)) { - return Reply( - std::move(response), - Ydb::StatusIds::BAD_REQUEST, - TStringBuilder() << "Failed item check: " << explain - ); + if (!Prepare(buildInfo, domainPath, tablePath, settings, explain)) { + return Reply(std::move(response), Ydb::StatusIds::BAD_REQUEST, TStringBuilder() + << "Failed item check: " << explain); + } + + const auto indexDesc = buildInfo->SerializeToProto(Self).GetIndex(); + if (!NTableIndex::CommonCheck(tableInfo, indexDesc, domainInfo->GetSchemeLimits(), explain)) { + return Reply(std::move(response), Ydb::StatusIds::BAD_REQUEST, explain); } Y_VERIFY(buildInfo != nullptr); @@ -231,5 +245,4 @@ ITransaction* TSchemeShard::CreateTxCreate(TEvIndexBuilder::TEvCreateRequest::TP return new TIndexBuilder::TTxCreate(this, ev); } -} // NSchemeShard -} // NKikimr +} diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 1ca5ec73285..2334a721df7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -14,23 +14,6 @@ namespace NKikimr { namespace NSchemeShard { -NKikimrSchemeOp::TIndexBuildConfig GetInitiateIndexBuildMessage(TSchemeShard* ss, const TIndexBuildInfo::TPtr buildInfo) { - NKikimrSchemeOp::TIndexBuildConfig message; - - message.SetTable(TPath::Init(buildInfo->TablePathId, ss).PathString()); - auto& index = *message.MutableIndex(); - index.SetName(buildInfo->IndexName); - for (const auto& x: buildInfo->IndexColumns) { - *index.AddKeyColumnNames() = x; - } - for (const auto& x: buildInfo->DataColumns) { - *index.AddDataColumnNames() = x; - } - index.SetType(buildInfo->IndexType); - - return message; -} - THolder<TEvSchemeShard::TEvModifySchemeTransaction> LockPropose( TSchemeShard* ss, const TIndexBuildInfo::TPtr buildInfo) { @@ -38,7 +21,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> LockPropose( propose->Record.SetFailOnExist(false); NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); - modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateLockForIndexBuild); + modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateLock); modifyScheme.SetInternal(true); TPath path = TPath::Init(buildInfo->TablePathId, ss); @@ -47,7 +30,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> LockPropose( auto& lockConfig = *modifyScheme.MutableLockConfig(); lockConfig.SetName(path.LeafName()); - *modifyScheme.MutableInitiateIndexBuild() = GetInitiateIndexBuildMessage(ss, buildInfo); + *modifyScheme.MutableInitiateIndexBuild() = buildInfo->SerializeToProto(ss); return propose; } @@ -66,7 +49,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> InitiatePropose( modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo->LockTxId)); - *modifyScheme.MutableInitiateIndexBuild() = GetInitiateIndexBuildMessage(ss, buildInfo); + *modifyScheme.MutableInitiateIndexBuild() = buildInfo->SerializeToProto(ss); return propose; } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 4ef795f29dc..3c265c9dd54 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -1287,7 +1287,7 @@ TPathElement::EPathState TSchemeShard::CalcPathState(TTxState::ETxType txType, T case TTxState::TxAlterUserAttributes: case TTxState::TxInitializeBuildIndex: case TTxState::TxFinalizeBuildIndex: - case TTxState::TxCreateLockForIndexBuild: + case TTxState::TxCreateLock: case TTxState::TxDropLock: case TTxState::TxAlterTableIndex: case TTxState::TxAlterSolomonVolume: diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 01985b866ce..5f17890af45 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -1,4 +1,5 @@ #include "schemeshard_info_types.h" +#include "schemeshard_path.h" #include "schemeshard_utils.h" #include <ydb/core/base/appdata.h> @@ -1714,6 +1715,25 @@ TIndexBuildInfo::TShardStatus::TShardStatus(TSerializedTableRange range, TString , LastKeyAck(std::move(lastKeyAck)) {} +NKikimrSchemeOp::TIndexBuildConfig TIndexBuildInfo::SerializeToProto(TSchemeShard* ss) const { + NKikimrSchemeOp::TIndexBuildConfig result; + result.SetTable(TPath::Init(TablePathId, ss).PathString()); + + auto& index = *result.MutableIndex(); + index.SetName(IndexName); + index.SetType(IndexType); + + for (const auto& x : IndexColumns) { + *index.AddKeyColumnNames() = x; + } + + for (const auto& x : DataColumns) { + *index.AddDataColumnNames() = x; + } + + return result; +} + TColumnFamiliesMerger::TColumnFamiliesMerger(NKikimrSchemeOp::TPartitionConfig &container) : Container(container) , DeduplicationById(TPartitionConfigMerger::DeduplicateColumnFamiliesById(Container)) diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index e9845105bd3..e3ddc608aa9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -33,6 +33,8 @@ namespace NKikimr { namespace NSchemeShard { +class TSchemeShard; + struct TForceShardSplitSettings { ui64 ForceShardSplitDataSize; bool DisableForceShardSplit; @@ -2782,6 +2784,8 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> { // No shards - no progress return 0.0; } + + NKikimrSchemeOp::TIndexBuildConfig SerializeToProto(TSchemeShard* ss) const; }; bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, diff --git a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h index c1dc7fbe913..2bc4a3b689d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tx_infly.h +++ b/ydb/core/tx/schemeshard/schemeshard_tx_infly.h @@ -81,7 +81,7 @@ struct TTxState { item(TxUpgradeSubDomain, 35) \ item(TxUpgradeSubDomainDecision, 36) \ item(TxInitializeBuildIndex, 37) \ - item(TxCreateLockForIndexBuild, 38) \ + item(TxCreateLock, 38) \ item(TxAlterTableIndex, 39) \ item(TxFinalizeBuildIndex, 40) \ item(TxAlterSolomonVolume, 41) \ @@ -326,7 +326,7 @@ struct TTxState { case TxInitializeBuildIndex: //this is more like alter case TxCreateCdcStreamAtTable: return false; - case TxCreateLockForIndexBuild: //this is more like alter + case TxCreateLock: //this is more like alter case TxDropLock: //this is more like alter return false; case TxDropTable: @@ -427,7 +427,7 @@ struct TTxState { case TxCreateReplication: case TxCreateBlobDepot: case TxInitializeBuildIndex: - case TxCreateLockForIndexBuild: + case TxCreateLock: case TxDropLock: case TxFinalizeBuildIndex: case TxDropTableIndexAtMainTable: // just increments schemaversion at main table @@ -513,7 +513,7 @@ struct TTxState { case TxCreateReplication: case TxCreateBlobDepot: case TxInitializeBuildIndex: - case TxCreateLockForIndexBuild: + case TxCreateLock: case TxDropLock: case TxDropTableIndexAtMainTable: case TxDropCdcStreamAtTable: diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp index f061c18e03d..44618276f98 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_utils.cpp @@ -498,14 +498,14 @@ bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesrc, TCol return true; } -TColumnTypes ExtractTypes(const NSchemeShard::TTableInfo::TPtr& baseTableInfo) { - TColumnTypes columsTypes; - for (auto& column: baseTableInfo->Columns) { - auto& columnName = column.second.Name; - columsTypes[columnName] = column.second.PType; +bool ExtractTypes(const NSchemeShard::TTableInfo::TPtr& baseTableInfo, TColumnTypes& columsTypes, TString& explain) { + Y_UNUSED(explain); + + for (const auto& [_, column] : baseTableInfo->Columns) { + columsTypes[column.Name] = column.PType; } - return columsTypes; + return true; } bool IsCompatibleKeyTypes( diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.h b/ydb/core/tx/schemeshard/schemeshard_utils.h index 684b3f94ef9..2569be3e8af 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.h +++ b/ydb/core/tx/schemeshard/schemeshard_utils.h @@ -123,7 +123,7 @@ private: bool SelfPingWakeupScheduled; }; -} +} // NSchemeShard namespace NTableIndex { @@ -133,7 +133,7 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc( const NKikimrSchemeOp::TTableDescription& indexTableDesc); NKikimrSchemeOp::TTableDescription CalcImplTableDesc( - const NKikimrSchemeOp::TTableDescription& baseTableDesrc, + const NKikimrSchemeOp::TTableDescription& baseTableDesc, const NTableIndex::TTableColumns& implTableColumns, const NKikimrSchemeOp::TTableDescription& indexTableDesc); @@ -142,23 +142,82 @@ NKikimrSchemeOp::TPartitionConfig PartitionConfigForIndexes( const NKikimrSchemeOp::TTableDescription& indexTableDesc); NKikimrSchemeOp::TPartitionConfig PartitionConfigForIndexes( - const NKikimrSchemeOp::TTableDescription& baseTableDesrc, + const NKikimrSchemeOp::TTableDescription& baseTableDesc, const NKikimrSchemeOp::TTableDescription& indexTableDesc); -TTableColumns ExtractInfo(const NKikimrSchemeOp::TTableDescription& tableDesrc); -TIndexColumns ExtractInfo(const NKikimrSchemeOp::TIndexCreationConfig& indexDesc); TTableColumns ExtractInfo(const NSchemeShard::TTableInfo::TPtr& tableInfo); +TTableColumns ExtractInfo(const NKikimrSchemeOp::TTableDescription& tableDesc); +TIndexColumns ExtractInfo(const NKikimrSchemeOp::TIndexCreationConfig& indexDesc); using TColumnTypes = THashMap<TString, NScheme::TTypeInfo>; -bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesrc, TColumnTypes& columsTypes, TString& explain); -TColumnTypes ExtractTypes(const NSchemeShard::TTableInfo::TPtr& baseTableInfo); +bool ExtractTypes(const NSchemeShard::TTableInfo::TPtr& baseTableInfo, TColumnTypes& columsTypes, TString& explain); +bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesc, TColumnTypes& columsTypes, TString& explain); bool IsCompatibleKeyTypes( const TColumnTypes& baseTableColumsTypes, const TTableColumns& implTableColumns, bool uniformTable, TString& explain); + +template <typename TTableDesc> +bool CommonCheck(const TTableDesc& tableDesc, const NKikimrSchemeOp::TIndexCreationConfig& indexDesc, + const NSchemeShard::TSchemeLimits& schemeLimits, bool uniformTable, + TTableColumns& implTableColumns, NKikimrScheme::EStatus& status, TString& error) +{ + const TTableColumns baseTableColumns = ExtractInfo(tableDesc); + const TIndexColumns indexKeys = ExtractInfo(indexDesc); + + if (indexKeys.KeyColumns.empty()) { + status = NKikimrScheme::EStatus::StatusInvalidParameter; + error = "No key colums in index creation config"; + return false; + } + + if (!indexKeys.DataColumns.empty() && !AppData()->FeatureFlags.GetEnableDataColumnForIndexTable()) { + status = NKikimrScheme::EStatus::StatusPreconditionFailed; + error = "It is not allowed to create index with data column"; + return false; + } + + if (!IsCompatibleIndex(baseTableColumns, indexKeys, error)) { + status = NKikimrScheme::EStatus::StatusInvalidParameter; + return false; + } + + TColumnTypes columnsTypes; + if (!ExtractTypes(tableDesc, columnsTypes, error)) { + status = NKikimrScheme::EStatus::StatusInvalidParameter; + return false; + } + + implTableColumns = CalcTableImplDescription(baseTableColumns, indexKeys); + if (!IsCompatibleKeyTypes(columnsTypes, implTableColumns, uniformTable, error)) { + status = NKikimrScheme::EStatus::StatusInvalidParameter; + return false; + } + + if (implTableColumns.Keys.size() > schemeLimits.MaxTableKeyColumns) { + status = NKikimrScheme::EStatus::StatusSchemeError; + error = TStringBuilder() + << "Too many keys indexed, index table reaches the limit of the maximum key colums count" + << ": indexing colums: " << indexKeys.KeyColumns.size() + << ", requested keys colums for index table: " << implTableColumns.Keys.size() + << ", limit: " << schemeLimits.MaxTableKeyColumns; + return false; + } + + return true; } +template <typename TTableDesc> +bool CommonCheck(const TTableDesc& tableDesc, const NKikimrSchemeOp::TIndexCreationConfig& indexDesc, + const NSchemeShard::TSchemeLimits& schemeLimits, TString& error) +{ + TTableColumns implTableColumns; + NKikimrScheme::EStatus status; + return CommonCheck(tableDesc, indexDesc, schemeLimits, false, implTableColumns, status, error); } + +} // NTableIndex +} // NKikimr diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index f0558cacd95..ce4f3c84155 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -436,7 +436,7 @@ namespace NSchemeShardUT_Private { TEvSchemeShard::TEvModifySchemeTransaction* LockRequest(ui64 txId, const TString &parentPath, const TString& name) { THolder<TEvSchemeShard::TEvModifySchemeTransaction> evTx = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(txId, TTestTxConfig::SchemeShard); auto transaction = evTx->Record.AddTransaction(); - transaction->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateLockForIndexBuild); + transaction->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateLock); transaction->SetWorkingDir(parentPath); auto op = transaction->MutableLockConfig(); op->SetName(name); diff --git a/ydb/core/tx/tx_proxy/schemereq.cpp b/ydb/core/tx/tx_proxy/schemereq.cpp index ff29572d944..3a697a9eeab 100644 --- a/ydb/core/tx/tx_proxy/schemereq.cpp +++ b/ydb/core/tx/tx_proxy/schemereq.cpp @@ -211,8 +211,8 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOpInitiateBuildIndexMainTable: Y_FAIL("no implementation for ESchemeOpInitiateBuildIndexMainTable"); - case NKikimrSchemeOp::ESchemeOpCreateLockForIndexBuild: - Y_FAIL("no implementation for ESchemeOpCreateLockForIndexBuild"); + case NKikimrSchemeOp::ESchemeOpCreateLock: + Y_FAIL("no implementation for ESchemeOpCreateLock"); case NKikimrSchemeOp::ESchemeOpApplyIndexBuild: Y_FAIL("no implementation for ESchemeOpApplyIndexBuild"); @@ -712,7 +712,7 @@ struct TBaseSchemeReq: public TActorBootstrapped<TDerived> { case NKikimrSchemeOp::ESchemeOp_DEPRECATED_35: case NKikimrSchemeOp::ESchemeOpCreateIndexBuild: case NKikimrSchemeOp::ESchemeOpInitiateBuildIndexMainTable: - case NKikimrSchemeOp::ESchemeOpCreateLockForIndexBuild: + case NKikimrSchemeOp::ESchemeOpCreateLock: case NKikimrSchemeOp::ESchemeOpApplyIndexBuild: case NKikimrSchemeOp::ESchemeOpFinalizeBuildIndexMainTable: case NKikimrSchemeOp::ESchemeOpAlterTableIndex: |