diff options
author | serg-belyakov <serg-belyakov@yandex-team.com> | 2022-09-19 11:25:40 +0300 |
---|---|---|
committer | serg-belyakov <serg-belyakov@yandex-team.com> | 2022-09-19 11:25:40 +0300 |
commit | c261c7112a1857f7c093dd87cfd07a93cd219822 (patch) | |
tree | 8cffe1c58a2b6f522863997c963dc56a78dfbf05 | |
parent | b8ba0035bc558ad4ec20e5e24447a5f20a7ee221 (diff) | |
download | ydb-c261c7112a1857f7c093dd87cfd07a93cd219822.tar.gz |
Add multiple PDisk main keys,
Add pdisk-key option to CLI
Fix key obtaining
Key change now works
Change pdisk main key type to TMainKey (typedefed as TStackVec<NPDisk::TKey, 2>)
36 files changed, 549 insertions, 240 deletions
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index d8bc5b3529c..a982ae15bd9 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -687,11 +687,12 @@ struct TEvBlobStorage { EvNonrestoredCorruptedBlobNotify, EvHugeLockChunks, EvHugeStat, - EvForwardToSkeleton, + EvForwardToSkeleton, // 268 636 300 EvHugeUnlockChunks, EvVDiskStatRequest, EvGetLogoBlobRequest, EvChunkForget, + EvFormatReencryptionFinish, EvYardInitResult = EvPut + 9 * 512, /// 268 636 672 EvLogResult, diff --git a/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp b/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp index 69a436bdeaf..c74015e50ac 100644 --- a/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp +++ b/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp @@ -129,6 +129,7 @@ class TQueueTestRuntime { ui32 ChunkSize; ui64 PDiskGuid; NPDisk::TKey PDiskKey; + NPDisk::TMainKey MainKey; TActorId PDiskId; std::unique_ptr<TAppData> AppData; TVDiskID VDiskId; @@ -160,7 +161,8 @@ public: DiskSize = SectorMap->DeviceSize; PDiskGuid = 1; PDiskKey = 1; - FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, PDiskKey, "queue_test", + MainKey = {1}; + FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, MainKey.back(), "queue_test", false, false, SectorMap); PDiskId = MakeBlobStoragePDiskID(1, 1); @@ -170,7 +172,7 @@ public: pDiskConfig->WriteCacheSwitch = NKikimrBlobStorage::TPDiskConfig::DoNotTouch; pDiskConfig->SectorMap = SectorMap; pDiskConfig->EnableSectorEncryption = !pDiskConfig->SectorMap; - TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), PDiskKey, Counters), TMailboxType::Revolving, 0); + TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), MainKey, Counters), TMailboxType::Revolving, 0); setup->LocalServices.emplace_back(PDiskId, pDiskSetup); //////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/core/blobstorage/base/blobstorage_events.h b/ydb/core/blobstorage/base/blobstorage_events.h index 7e7fa781eed..b653832457e 100644 --- a/ydb/core/blobstorage/base/blobstorage_events.h +++ b/ydb/core/blobstorage/base/blobstorage_events.h @@ -437,10 +437,10 @@ namespace NKikimr { struct TEvBlobStorage::TEvRestartPDisk : TEventLocal<TEvRestartPDisk, EvRestartPDisk> { const ui32 PDiskId; - NPDisk::TKey MainKey; + const NPDisk::TMainKey MainKey; TIntrusivePtr<TPDiskConfig> Config; - TEvRestartPDisk(const ui32& pdiskId, const NPDisk::TKey& mainKey, const TIntrusivePtr<TPDiskConfig>& config) + TEvRestartPDisk(const ui32& pdiskId, const NPDisk::TMainKey& mainKey, const TIntrusivePtr<TPDiskConfig>& config) : PDiskId(pdiskId) , MainKey(mainKey) , Config(config) diff --git a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp index 07f5948f3ae..1d881f19f01 100644 --- a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp @@ -4227,8 +4227,9 @@ public: pDiskConfig->SectorMap = SectorMapByPath[filePath]; pDiskConfig->EnableSectorEncryption = !pDiskConfig->SectorMap; + NPDisk::TMainKey mainKeys = { mainKey }; TActorSetupCmd pDiskSetup( - CreatePDisk(pDiskConfig.Get(), mainKey, counters), + CreatePDisk(pDiskConfig.Get(), mainKeys, counters), TMailboxType::Revolving, 0); setup2->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(pdiskId, pDiskSetup)); diff --git a/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp b/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp index ae065199ff5..a4f8b3e15f5 100644 --- a/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp +++ b/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp @@ -29,6 +29,7 @@ public: ui64 DiskSize; ui64 PDiskGuid; ui64 PDiskKey; + NPDisk::TMainKey MainKey; TActorId PDiskId; TActorId KeeperId; std::unique_ptr<TActorSystem> ActorSystem; @@ -57,13 +58,14 @@ public: DiskSize = (ui64)ChunkSize * numChunks; PDiskGuid = Now().GetValue(); PDiskKey = 1; - FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, PDiskKey, "incrhuge"); + MainKey = {1}; + FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, MainKey.back(), "incrhuge"); } PDiskId = MakeBlobStoragePDiskID(1, 1); ui64 pDiskCategory = 0; TIntrusivePtr<TPDiskConfig> pDiskConfig = new TPDiskConfig(Path, PDiskGuid, 1, pDiskCategory); - TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), PDiskKey, Counters), TMailboxType::Revolving, 0); + TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), MainKey, Counters), TMailboxType::Revolving, 0); setup->LocalServices.emplace_back(PDiskId, pDiskSetup); TActorId pdiskActorId; diff --git a/ydb/core/blobstorage/nodewarden/node_warden.h b/ydb/core/blobstorage/nodewarden/node_warden.h index 9ecc35bd3c1..fb1cfe370c5 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden.h +++ b/ydb/core/blobstorage/nodewarden/node_warden.h @@ -28,7 +28,7 @@ namespace NKikimr { std::unique_ptr<ICacheAccessor> CacheAccessor; TEncryptionKey TenantKey; TEncryptionKey StaticKey; - TEncryptionKey PDiskKey; + TVector<TEncryptionKey> PDiskKey; bool CachePDisks = false; bool CacheVDisks = false; bool EnableVDiskCooldownTimeout = false; @@ -42,15 +42,19 @@ namespace NKikimr { , AllDriveModels(new NPDisk::TDriveModelDb) {} - NPDisk::TKey CreatePDiskKey() const { - if (PDiskKey) { - const ui8 *key; - ui32 keySize; - PDiskKey.Key.GetKeyBytes(&key, &keySize); - return *(ui64*)key; - } else { - return NPDisk::YdbDefaultPDiskSequence; - } + NPDisk::TMainKey CreatePDiskKey() const { + if (PDiskKey.size() > 0) { + NPDisk::TMainKey mainKey; + for (ui32 i = 0; i < PDiskKey.size(); ++i) { + const ui8 *key; + ui32 keySize; + PDiskKey[i].Key.GetKeyBytes(&key, &keySize); + mainKey.push_back(*(ui64*)key); + } + return mainKey; + } else { + return { NPDisk::YdbDefaultPDiskSequence }; + } } bool IsCacheEnabled() const { @@ -62,7 +66,7 @@ namespace NKikimr { bool ObtainTenantKey(TEncryptionKey *key, const NKikimrProto::TKeyConfig& keyConfig); bool ObtainStaticKey(TEncryptionKey *key); - bool ObtainPDiskKey(TEncryptionKey *key, const NKikimrProto::TKeyConfig& keyConfig); + bool ObtainPDiskKey(TVector<TEncryptionKey> *key, const NKikimrProto::TKeyConfig& keyConfig); std::unique_ptr<ICacheAccessor> CreateFileCacheAccessor(const TString& templ, const std::unordered_map<char, TString>& vars); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index 02586ae4506..4457106f62c 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -489,14 +489,38 @@ bool NKikimr::ObtainTenantKey(TEncryptionKey *key, const NKikimrProto::TKeyConfi } } -bool NKikimr::ObtainPDiskKey(TEncryptionKey *key, const NKikimrProto::TKeyConfig& keyConfig) { - if (keyConfig.KeysSize()) { - auto &record = keyConfig.GetKeys(0); - return ObtainKey(key, record); - } else { +bool NKikimr::ObtainPDiskKey(TVector<TEncryptionKey> *keys, const NKikimrProto::TKeyConfig& keyConfig) { + ui32 keysSize = keyConfig.KeysSize(); + if (!keysSize) { Cerr << "No Keys in PDiskKeyConfig! Encrypted pdisks will not start" << Endl; return false; } + + keys->resize(keysSize); + for (ui32 i = 0; i < keysSize; ++i) { + auto &record = keyConfig.GetKeys(i); + if (record.GetId() == "0" && record.GetContainerPath() == "") { + // use default pdisk key + (*keys)[i].Id = "0"; + (*keys)[i].Version = record.GetVersion(); + + ui8 *keyBytes = 0; + ui32 keySize = 0; + (*keys)[i].Key.MutableKeyBytes(&keyBytes, &keySize); + + ui64* p = (ui64*)keyBytes; + p[0] = NPDisk::YdbDefaultPDiskSequence; + } else { + if (!ObtainKey(&(*keys)[i], record)) { + return false; + } + } + } + + std::sort(keys->begin(), keys->end(), [&](const TEncryptionKey& l, const TEncryptionKey& r) { + return l.Version < r.Version; + }); + return true; } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp index edabea0c153..07a0b1e3ef7 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp @@ -107,12 +107,15 @@ namespace NKikimr::NStorage { pdiskConfig->EnableSectorEncryption = !pdiskConfig->SectorMap; } - NPDisk::TKey pdiskKey = Cfg->CreatePDiskKey(); - THashCalculator hasher; + NPDisk::TMainKey pdiskKey = Cfg->CreatePDiskKey(); TString keyPrintSalt = "@N2#_lW19)2-31!iifI@n1178349617"; - hasher.Hash(keyPrintSalt.Detach(), keyPrintSalt.Size()); - hasher.Hash(&pdiskKey, sizeof(pdiskKey)); - pdiskConfig->HashedMainKey = TStringBuilder() << Hex(hasher.GetHashResult(), HF_ADDX); + pdiskConfig->HashedMainKey.resize(pdiskKey.size()); + for (ui32 i = 0; i < pdiskKey.size(); ++i) { + THashCalculator hasher; + hasher.Hash(keyPrintSalt.Detach(), keyPrintSalt.Size()); + hasher.Hash(&pdiskKey[i], sizeof(pdiskKey[i])); + pdiskConfig->HashedMainKey[i] = TStringBuilder() << Hex(hasher.GetHashResult(), HF_ADDX); + } pdiskConfig->Initialize(); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 765a05d4782..62efca4a615 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -16,7 +16,7 @@ namespace NKikimr { -IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, +IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters); namespace NPDisk { @@ -124,7 +124,6 @@ public: } }; - //////////////////////////////////////////////////////////////////////////// // INIT //////////////////////////////////////////////////////////////////////////// diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp index 220748cd181..b3d89908b25 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp @@ -67,7 +67,7 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> { TString StateErrorReason; TIntrusivePtr<TPDiskConfig> Cfg; - TKey MainKey; + NPDisk::TMainKey MainKey; TList<TInitQueueItem> InitQueue; const TIntrusivePtr<::NMonitoring::TDynamicCounters> PDiskCounters; TIntrusivePtr<TPDisk> PDisk; @@ -195,7 +195,7 @@ public: return NKikimrServices::TActivity::PDISK_ACTOR; } - TPDiskActor(const TIntrusivePtr<TPDiskConfig>& cfg, const NPDisk::TKey &mainKey, + TPDiskActor(const TIntrusivePtr<TPDiskConfig>& cfg, const NPDisk::TMainKey& mainKey, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) : Cfg(cfg) , MainKey(mainKey) @@ -203,10 +203,11 @@ public: ->GetSubgroup("pdisk", Sprintf("%09" PRIu32, (ui32)cfg->PDiskId)) ->GetSubgroup("media", to_lower(cfg->PDiskCategory.TypeStrShort()))) { + Y_VERIFY(!MainKey.empty()); } ~TPDiskActor() { - SecureWipeBuffer((ui8*)&MainKey, sizeof(MainKey)); + SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size()); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -362,7 +363,7 @@ public: try { FormatPDisk(cfg->GetDevicePath(), 0, cfg->SectorSize, cfg->ChunkSize, - cfg->PDiskGuid, chunkKey, logKey, sysLogKey, actor->MainKey, TString(), false, + cfg->PDiskGuid, chunkKey, logKey, sysLogKey, actor->MainKey.back(), TString(), false, cfg->FeatureFlags.GetTrimEntireDeviceOnStartup(), cfg->SectorMap); actorSystem->Send(pDiskActor, new TEvPDiskFormattingFinished(true, "")); } catch (yexception ex) { @@ -375,7 +376,7 @@ public: FormattingThread->Start(); } else { - SecureWipeBuffer((ui8*)&MainKey, sizeof(MainKey)); + SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size()); *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError; *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error; *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorPDiskCannotBeInitialised; @@ -386,55 +387,133 @@ public: } TStringStream str; str << "PDiskId# " << PDisk->PDiskId - << " Can not be initialized! " << PDisk->ErrorStr - << " Hash(MainKey)# " << Cfg->HashedMainKey; + << " Can not be initialized! " << PDisk->ErrorStr; + for (ui32 i = 0; i < Cfg->HashedMainKey.size(); ++i) { + str << " Hash(NewMainKey[" << i << "])# " << Cfg->HashedMainKey[i]; + } InitError(str.Str()); str << " Config: " << Cfg->ToString(); LOG_CRIT_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str()); } } + void ReencryptDiskFormat(const TDiskFormat& format, const NPDisk::TKey& newMainKey) { + IsFormattingNow = true; + // Stop PDiskThread but use PDisk object for creation of http pages + PDisk->Stop(); + *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingReencryptingFormat; + PDisk->ErrorStr = " Format sectors are encrypted with an old PDisk key, reencryption started"; + LOG_WARN_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId << PDisk->ErrorStr); + + // Is used to pass parameters into formatting thread, because TThread can pass only void* + using TCookieType = std::tuple<TDiskFormat, NPDisk::TKey, TIntrusivePtr<TPDiskConfig>, NActors::TActorSystem*, TActorId>; + FormattingThread.Reset(new TThread( + [] (void *cookie) -> void* { + std::unique_ptr<TCookieType> params(static_cast<TCookieType*>(cookie)); + TDiskFormat format = std::get<0>(*params); + NPDisk::TKey mainKey = std::get<1>(*params); + TIntrusivePtr<TPDiskConfig> cfg = std::get<2>(*params); + const TIntrusivePtr<::NMonitoring::TDynamicCounters> counters(new ::NMonitoring::TDynamicCounters); + NActors::TActorSystem* actorSystem = std::get<3>(*params); + TActorId pdiskActor = std::get<4>(*params); + + THolder<NPDisk::TPDisk> pDisk(new NPDisk::TPDisk(cfg, counters)); + + pDisk->Initialize(actorSystem, TActorId()); + + if (!pDisk->BlockDevice->IsGood()) { + ythrow yexception() << "Failed to initialize temporal PDisk for format rewriting, info# " << pDisk->BlockDevice->DebugInfo(); + } + + try { + pDisk->WriteApplyFormatRecord(format, mainKey); + actorSystem->Send(pdiskActor, new TEvFormatReencryptionFinish(true, "")); + } catch (yexception ex) { + LOG_ERROR_S(*actorSystem, NKikimrServices::BS_PDISK, "Reencryption error, what#" << ex.what()); + actorSystem->Send(pdiskActor, new TEvFormatReencryptionFinish(false, ex.what())); + } + return nullptr; + }, + new TCookieType(format, newMainKey, PDisk->Cfg, TlsActivationContext->ActorSystem(), SelfId()) + )); + FormattingThread->Start(); + } + + void InitHandle(TEvFormatReencryptionFinish::TPtr &ev) { + FormattingThread->Join(); + IsFormattingNow = false; + if (ev->Get()->Success) { + StartPDiskThread(); + LOG_WARN_S(*TlsActivationContext, NKikimrServices::BS_PDISK, + "PDiskId# " << PDisk->PDiskId << " format chunks reencryption finished"); + } else { + PDisk.Reset(new TPDisk(Cfg, PDiskCounters)); + PDisk->Initialize(TlsActivationContext->ActorSystem(), SelfId()); + Y_VERIFY(PDisk->PDiskThread.Running()); + + *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError; + *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error; + *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorDiskCannotBeFormated; + + PDisk->ErrorStr = ToString("Format chunks cannot be reencrypted! Reason# ") + ev->Get()->ErrorReason; + + TStringStream str; + str << "PDiskId# " << (ui32)PDisk->PDiskId + << " Format chunks cannot be reencrypted! Reason# " << ev->Get()->ErrorReason + << " Switching to StateError. Config: " << Cfg->ToString(); + LOG_CRIT_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str()); + InitError(str.Str()); + } + } + void InitHandle(TEvReadFormatResult::TPtr &ev) { ui8 *formatSectors = ev->Get()->FormatSectors.Get(); ui32 formatSectorsSize = ev->Get()->FormatSectorsSize; NSan::CheckMemIsInitialized(formatSectors, formatSectorsSize); - bool isFormatOk = PDisk->ReadChunk0Format(formatSectors, MainKey); - if (!isFormatOk) { + TCheckDiskFormatResult res = PDisk->ReadChunk0Format(formatSectors, MainKey); + if (!res.IsFormatPresent) { *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingFormatMagicChecking; - PDisk->ErrorStr = "Format is not Ok, now checking for proper magic sector on disk"; + PDisk->ErrorStr = "Format chunks are not present on disk or corrupted, now checking for proper magic sector on disk"; CheckMagicSector(formatSectors, formatSectorsSize); } else { - SecureWipeBuffer((ui8*)&MainKey, sizeof(MainKey)); - // Format is read OK - LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId - << " Successfully read format record# " << PDisk->Format.ToString()); - TString info; - if (!PDisk->CheckGuid(&info)) { - *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError; - *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error; - *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadDueToGuid; - PDisk->ErrorStr = TStringBuilder() << "Can't start due to a guid error " << info; - TStringStream str; - str << "PDiskId# " << PDisk->PDiskId << PDisk->ErrorStr; - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str()); - InitError(str.Str()); - } else if (!PDisk->CheckFormatComplete()) { - *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError; - *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error; - *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadIncompleteFormat; - PDisk->ErrorStr = "Can't start due to incomplete format!"; - TStringStream str; - str << "PDiskId# " << PDisk->PDiskId << " " << PDisk->ErrorStr << " " - << "Please, do not trun off your server or remove your storage device while formatting. " - << "We are sure you did this or something even more creative, like killing the formatter."; - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str()); - InitError(str.Str()); + if (res.IsReencryptionRequired) { + // Format reencryption required + ReencryptDiskFormat(PDisk->Format, MainKey.back()); + // We still need main key after restart + // SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size()); } else { - // PDisk GUID is OK and format is complete - *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialSysLogRead; - *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingSysLogRead; - PDisk->Format.InitMagic(); - PDisk->ReadSysLog(SelfId()); + // Format is read OK + SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size()); + LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId + << " Successfully read format record# " << PDisk->Format.ToString()); + TString info; + if (!PDisk->CheckGuid(&info)) { + *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError; + *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error; + *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadDueToGuid; + PDisk->ErrorStr = TStringBuilder() << "Can't start due to a guid error " << info; + TStringStream str; + str << "PDiskId# " << PDisk->PDiskId << PDisk->ErrorStr; + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str()); + InitError(str.Str()); + } else if (!PDisk->CheckFormatComplete()) { + *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError; + *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error; + *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadIncompleteFormat; + PDisk->ErrorStr = "Can't start due to incomplete format!"; + TStringStream str; + str << "PDiskId# " << PDisk->PDiskId << " " << PDisk->ErrorStr << " " + << "Please, do not turn off your server or remove your storage device while formatting. " + << "We are sure you did this or something even more creative, like killing the formatter."; + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str()); + InitError(str.Str()); + } else { + // PDisk GUID is OK and format is complete + *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialSysLogRead; + *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingSysLogRead; + PDisk->Format.InitMagic(); + PDisk->ReadSysLog(SelfId()); + } } } } @@ -638,7 +717,7 @@ public: switch (evControl.Action) { case TEvYardControl::PDiskStart: { - auto *mainKey = static_cast<const NPDisk::TKey*>(evControl.Cookie); + auto *mainKey = static_cast<const NPDisk::TMainKey*>(evControl.Cookie); Y_VERIFY(mainKey); MainKey = *mainKey; StartPDiskThread(); @@ -879,7 +958,7 @@ public: } MainKey = ev->Get()->MainKey; - SecureWipeBuffer((ui8*)&ev->Get()->MainKey, sizeof(ev->Get()->MainKey)); + SecureWipeBuffer((ui8*)ev->Get()->MainKey.data(), sizeof(NPDisk::TKey) * ev->Get()->MainKey.size()); LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId << " Going to restart PDisk since recieved TEvRestartPDisk"); PDisk->Stop(); @@ -1084,6 +1163,7 @@ public: cFunc(TEvents::TSystem::Wakeup, HandleWakeup); hFunc(NPDisk::TEvDeviceError, Handle); hFunc(TEvBlobStorage::TEvRestartPDisk, Handle); + hFunc(NPDisk::TEvFormatReencryptionFinish, InitHandle); ) STRICT_STFUNC(StateOnline, @@ -1149,13 +1229,13 @@ public: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // PDisk Creation //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, +IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) { return new NPDisk::TPDiskActor(cfg, mainKey, counters); } void TRealPDiskServiceFactory::Create(const TActorContext &ctx, ui32 pDiskID, - const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) { + const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) { TActorId actorId = ctx.ExecutorThread.RegisterActor( CreatePDisk(cfg, mainKey, AppData(ctx)->Counters), TMailboxType::ReadAsFilled, poolId); TActorId pDiskServiceId = MakeBlobStoragePDiskID(nodeId, pDiskID); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h index 5e861add10d..0d1ca44e052 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h @@ -335,5 +335,34 @@ public: } }; +class TCompletionSequence : public TCompletionAction { + TVector<TCompletionAction*> Actions; + +public: + TCompletionSequence() = default; + TCompletionSequence(const TVector<TCompletionAction*>& actions) + : Actions(actions) + {} + + TCompletionSequence(TVector<TCompletionAction*>&& actions) + : Actions(std::move(actions)) + {} + + TCompletionSequence& operator=(const TCompletionSequence&) = delete; + TCompletionSequence& operator=(TCompletionSequence&&) = delete; + + void Exec(TActorSystem *actorSystem) override { + for (TCompletionAction* action : Actions) { + action->Exec(actorSystem); + } + } + + void Release(TActorSystem *actorSystem) override { + for (TCompletionAction* action : Actions) { + action->Release(actorSystem); + } + } +}; + } // NPDisk } // NKikimr diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h index 5990a2ee874..94440f6c8ee 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h @@ -89,7 +89,7 @@ struct TPDiskConfig : public TThrRefBase { ui64 PDiskGuid; // set only by constructor ui32 PDiskId; // set only by constructor TPDiskCategory PDiskCategory; // set only by constructor - TString HashedMainKey; + TStackVec<TString, 2> HashedMainKey; ui64 StartOwnerRound = 1ull; // set only by warden TIntrusivePtr<NPDisk::TSectorMap> SectorMap; // set only by warden @@ -244,7 +244,9 @@ struct TPDiskConfig : public TThrRefBase { str << " PDiskGuid# " << PDiskGuid << x; str << " PDiskId# " << PDiskId << x; str << " PDiskCategory# " << PDiskCategory.ToString() << x; - str << " HashedMainKey# " << HashedMainKey << x; + for (ui32 i = 0; i < HashedMainKey.size(); ++i) { + str << " HashedMainKey[" << i << "]# " << HashedMainKey[i] << x; + } str << " StartOwnerRound# " << StartOwnerRound << x; str << " SectorMap# " << (SectorMap ? "true" : "false") << x; str << " EnableSectorEncryption # " << EnableSectorEncryption << x; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h index 2b1350bff79..c80fd915cd9 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h @@ -777,6 +777,16 @@ union TDiskFormatSector { static_assert(sizeof(TDiskFormat) <= FormatSectorSize, "TDiskFormat size too lagre!"); +struct TCheckDiskFormatResult { + bool IsFormatPresent; + bool IsReencryptionRequired; + + TCheckDiskFormatResult(bool isFormatPresent, bool isReencryptionRequired) + : IsFormatPresent(isFormatPresent) + , IsReencryptionRequired(isReencryptionRequired) + {} +}; + } // NPDisk } // NKikimr diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h index cc63207fc7f..8b1fb2c8fc7 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h @@ -38,6 +38,7 @@ namespace NKikimr { typedef ui32 TStatusFlags; typedef ui64 TKey; typedef ui64 THash; + typedef TStackVec<TKey, 2> TMainKey; struct TOwnerToken { TOwner Owner = 0; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h index 6d472eab412..9c2886e2081 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h @@ -13,13 +13,13 @@ namespace NKikimr { class IPDiskServiceFactory : public TThrRefBase { public: virtual void Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg, - const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) = 0; + const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) = 0; }; class TRealPDiskServiceFactory : public IPDiskServiceFactory { public: void Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg, - const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) override; + const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) override; }; } // NKikimr diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index d7e554675c3..14e876426d8 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -101,80 +101,85 @@ TString TPDisk::DynamicStateToString(bool isMultiline) { return str.Str(); } -bool TPDisk::ReadChunk0Format(ui8* formatSectors, const TKey& mainKey) { +TCheckDiskFormatResult TPDisk::ReadChunk0Format(ui8* formatSectors, const NPDisk::TMainKey& mainKey) { TGuard<TMutex> guard(StateMutex); - TPDiskStreamCypher cypher(true); // Format record is always encrypted - cypher.SetKey(mainKey); - Format.SectorSize = FormatSectorSize; - ui32 lastGoodIdx = (ui32)-1; - bool isBad[ReplicationFactor]; - bool isBadPresent = false; - - for (ui32 i = 0; i < ReplicationFactor; ++i) { - ui64 sectorOffset = i * FormatSectorSize; - ui8* formatSector = formatSectors + sectorOffset; - TDataSectorFooter *footer = (TDataSectorFooter*) - (formatSector + FormatSectorSize - sizeof(TDataSectorFooter)); - - cypher.StartMessage(footer->Nonce); - alignas(16) TDiskFormat diskFormat; - cypher.Encrypt(&diskFormat, formatSector, sizeof(TDiskFormat)); - - isBad[i] = !diskFormat.IsHashOk(FormatSectorSize); - if (!isBad[i]) { - Format.UpgradeFrom(diskFormat); - if (Format.IsErasureEncodeUserChunks()) { - LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId - << " Read from disk Format has FormatFlagErasureEncodeUserChunks set, " - << " but current version of PDisk can't work with it" - << " Format# " << Format.ToString() - << " Marker# BPD80"); - Y_FAIL_S("PDiskId# " << PDiskId - << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserChunks set"); - } - if (Format.IsErasureEncodeUserLog()) { - LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId - << " Read from disk Format has FormatFlagErasureEncodeUserLog set, " - << " but current version of PDisk can't work with it" - << " Format# " << Format.ToString() - << " Marker# BPD801"); - Y_FAIL_S("PDiskId# " << PDiskId - << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserLog set"); - } - lastGoodIdx = i; - *Mon.TotalSpaceBytes = Format.DiskSize; - } else { - isBadPresent = true; - } - } - - if (lastGoodIdx < ReplicationFactor) { - ui64 sectorOffset = lastGoodIdx * FormatSectorSize; - ui8* formatSector = formatSectors + sectorOffset; - if (isBadPresent) { - for (ui32 i = 0; i < ReplicationFactor; ++i) { - if (isBad[i]) { - TBuffer* buffer = BufferPool->Pop(); - Y_VERIFY(FormatSectorSize <= buffer->Size()); - memcpy(buffer->Data(), formatSector, FormatSectorSize); - ui64 targetOffset = i * FormatSectorSize; - LOG_INFO_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << (ui32)PDiskId - << " PWriteAsync offset# " << targetOffset - << " to# " << (targetOffset + FormatSectorSize) - << " for format restoration" - << " Marker# BPD46"); - - REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer->Data(), FormatSectorSize); - BlockDevice->PwriteAsync(buffer->Data(), FormatSectorSize, targetOffset, buffer, - TReqId(TReqId::RestoreFormatOnRead, 0), {}); + Format.SectorSize = FormatSectorSize; + ui32 mainKeySize = mainKey.size(); + + for (ui32 k = 0; k < mainKeySize; ++k) { + TPDiskStreamCypher cypher(true); // Format record is always encrypted + cypher.SetKey(mainKey[k]); + + ui32 lastGoodIdx = (ui32)-1; + bool isBad[ReplicationFactor]; + bool isBadPresent = false; + for (ui32 i = 0; i < ReplicationFactor; ++i) { + ui64 sectorOffset = i * FormatSectorSize; + ui8* formatSector = formatSectors + sectorOffset; + TDataSectorFooter *footer = (TDataSectorFooter*) + (formatSector + FormatSectorSize - sizeof(TDataSectorFooter)); + + cypher.StartMessage(footer->Nonce); + alignas(16) TDiskFormat diskFormat; + cypher.Encrypt(&diskFormat, formatSector, sizeof(TDiskFormat)); + + isBad[i] = !diskFormat.IsHashOk(FormatSectorSize); + if (!isBad[i]) { + Format.UpgradeFrom(diskFormat); + if (Format.IsErasureEncodeUserChunks()) { + LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId + << " Read from disk Format has FormatFlagErasureEncodeUserChunks set, " + << " but current version of PDisk can't work with it" + << " Format# " << Format.ToString() + << " Marker# BPD80"); + Y_FAIL_S("PDiskId# " << PDiskId + << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserChunks set"); + } + if (Format.IsErasureEncodeUserLog()) { + LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId + << " Read from disk Format has FormatFlagErasureEncodeUserLog set, " + << " but current version of PDisk can't work with it" + << " Format# " << Format.ToString() + << " Marker# BPD801"); + Y_FAIL_S("PDiskId# " << PDiskId + << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserLog set"); } + lastGoodIdx = i; + *Mon.TotalSpaceBytes = Format.DiskSize; + } else { + isBadPresent = true; + } + } + if (lastGoodIdx < ReplicationFactor) { + ui64 sectorOffset = lastGoodIdx * FormatSectorSize; + ui8* formatSector = formatSectors + sectorOffset; + if (k < mainKeySize - 1) { // obsolete key is used + return TCheckDiskFormatResult(true, true); + } else if (isBadPresent) { + for (ui32 i = 0; i < ReplicationFactor; ++i) { + if (isBad[i]) { + TBuffer* buffer = BufferPool->Pop(); + Y_VERIFY(FormatSectorSize <= buffer->Size()); + memcpy(buffer->Data(), formatSector, FormatSectorSize); + ui64 targetOffset = i * FormatSectorSize; + LOG_INFO_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << (ui32)PDiskId + << " PWriteAsync offset# " << targetOffset + << " to# " << (targetOffset + FormatSectorSize) + << " for format restoration" + << " Marker# BPD46"); + + REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer->Data(), FormatSectorSize); + BlockDevice->PwriteAsync(buffer->Data(), FormatSectorSize, targetOffset, buffer, + TReqId(TReqId::RestoreFormatOnRead, 0), {}); + } + } + //BlockDevice->FlushAsync(nullptr); } - //BlockDevice->FlushAsync(nullptr); + return TCheckDiskFormatResult(true, false); } - return true; } - return false; + return TCheckDiskFormatResult(false, false); } bool TPDisk::IsFormatMagicValid(ui8 *magicData8, ui32 magicDataSize) { diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h index beeec00d4d1..c86b0a6be56 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h @@ -181,7 +181,7 @@ public: // Initialization TPDisk(const TIntrusivePtr<TPDiskConfig> cfg, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters); TString DynamicStateToString(bool isMultiline); - bool ReadChunk0Format(ui8* formatSectors, const TKey& mainKey); // Called by actor + TCheckDiskFormatResult ReadChunk0Format(ui8* formatSectors, const TMainKey& mainKey); // Called by actor bool IsFormatMagicValid(ui8 *magicData, ui32 magicDataSize); // Called by actor bool CheckGuid(TString *outReason); // Called by actor bool CheckFormatComplete(); // Called by actor @@ -190,6 +190,7 @@ public: void PrintChunksDebugInfo(); TString ProcessReadSysLogResult(ui64 &outWritePosition, ui64 &outLsn, const TEvReadLogResult &readLogResult); void ReadAndParseMainLog(const TActorId &pDiskActor); + void WriteFormatAsync(TDiskFormat format, const TKey &mainKey); // Called by the log reader on success with the current chunkOwnerMap. void ProcessChunkOwnerMap(TMap<ui32, TChunkState> &chunkOwnerMap); void InitLogChunksInfo(); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp index 0552ca8c88a..4f01fdad8b1 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp @@ -233,6 +233,7 @@ void TPDisk::ProcessChunk0(const NPDisk::TEvReadLogResult &readLogResult) { ui32 chunkCount = (ui32)(Format.DiskSize / (ui64)Format.ChunkSize); Y_VERIFY_DEBUG(ChunkState.size() == 0); ChunkState = TVector<TChunkState>(chunkCount); + Y_VERIFY(ChunkState.size() >= Format.SystemChunkCount); for (ui32 i = 0; i < Format.SystemChunkCount; ++i) { ChunkState[i].OwnerId = OwnerSystem; } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h index df2f0885694..e8a38d3eeeb 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h @@ -196,5 +196,31 @@ struct TEvDeviceError : public TEventLocal<TEvDeviceError, TEvBlobStorage::EvDev {} }; +struct TEvFormatReencryptionFinish : public TEventLocal<TEvFormatReencryptionFinish, TEvBlobStorage::EvFormatReencryptionFinish> { + bool Success; + TString ErrorReason; + + TEvFormatReencryptionFinish(bool success, TString errorReason) + : Success(success) + , ErrorReason(errorReason) + {} + + TString ToString() const { + return ToString(*this); + } + + static TString ToString(const TEvFormatReencryptionFinish& record) { + TStringStream str; + str << "{"; + str << "EvFormatReencryptionFinished "; + str << " Success# " << record.Success; + if (record.ErrorReason) { + str << " ErrorReason# " << record.ErrorReason; + } + str << "}"; + return str.Str(); + } +}; + } // NPDisk } // NKikimr diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h index 84fd08d685c..b5b10e63d44 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h @@ -361,6 +361,7 @@ struct TPDiskMon { ErrorNoDeviceWithSuchSerial, ErrorDeviceSerialMismatch, ErrorFake, + BootingReencryptingFormat, }; static TString StateToStr(i64 val) { @@ -402,6 +403,7 @@ struct TPDiskMon { case ErrorNoDeviceWithSuchSerial: return "ErrorNoDeviceWithSuchSerial"; case ErrorDeviceSerialMismatch: return "ErrorDeviceSerialMismatch"; case ErrorFake: return "ErrorFake"; + case BootingReencryptingFormat: return "BootingReencryptingFormat"; default: return "Unknown"; } } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp index 84f51fce940..34d062cce53 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp @@ -112,7 +112,7 @@ void FormatPDisk(TString path, ui64 diskSizeBytes, ui32 sectorSizeBytes, ui32 us chunkKey, logKey, sysLogKey, mainKey, textMessage, isErasureEncodeUserLog, trimEntireDevice); } -bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TKey &mainKey, TPDiskInfo &outInfo, +bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TMainKey &mainKey, TPDiskInfo &outInfo, const bool doLock, TIntrusivePtr<NPDisk::TSectorMap> sectorMap) { const TIntrusivePtr<::NMonitoring::TDynamicCounters> counters(new ::NMonitoring::TDynamicCounters); auto mon = std::make_unique<TPDiskMon>(counters, 0, nullptr); @@ -146,72 +146,74 @@ bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TKey &mainKey, TPDis blockDevice->PreadSync(formatRaw->Data(), formatSectorsSize, 0, NPDisk::TReqId(NPDisk::TReqId::ReadFormatInfo, 0), {}); - NPDisk::TPDiskStreamCypher cypher(true); // Format record is always encrypted - cypher.SetKey(mainKey); - bool isOk = false; - alignas(16) NPDisk::TDiskFormat format; - for (ui32 recordIdx = 0; recordIdx < NPDisk::ReplicationFactor; ++recordIdx) { - ui64 recordSectorOffset = recordIdx * NPDisk::FormatSectorSize; - ui8 *formatSector = formatRaw->Data() + recordSectorOffset; - NPDisk::TDataSectorFooter *footer = (NPDisk::TDataSectorFooter*) - (formatSector + NPDisk::FormatSectorSize - sizeof(NPDisk::TDataSectorFooter)); - - cypher.StartMessage(footer->Nonce); - - alignas(16) NPDisk::TDiskFormatSector formatCandidate; - cypher.Encrypt(formatCandidate.Raw, formatSector, NPDisk::FormatSectorSize); - - if (formatCandidate.Format.IsHashOk(NPDisk::FormatSectorSize)) { - format.UpgradeFrom(formatCandidate.Format); - isOk = true; + for (auto& key : mainKey) { + NPDisk::TPDiskStreamCypher cypher(true); // Format record is always encrypted + cypher.SetKey(key); + bool isOk = false; + alignas(16) NPDisk::TDiskFormat format; + for (ui32 recordIdx = 0; recordIdx < NPDisk::ReplicationFactor; ++recordIdx) { + ui64 recordSectorOffset = recordIdx * NPDisk::FormatSectorSize; + ui8 *formatSector = formatRaw->Data() + recordSectorOffset; + NPDisk::TDataSectorFooter *footer = (NPDisk::TDataSectorFooter*) + (formatSector + NPDisk::FormatSectorSize - sizeof(NPDisk::TDataSectorFooter)); + + cypher.StartMessage(footer->Nonce); + + alignas(16) NPDisk::TDiskFormatSector formatCandidate; + cypher.Encrypt(formatCandidate.Raw, formatSector, NPDisk::FormatSectorSize); + + if (formatCandidate.Format.IsHashOk(NPDisk::FormatSectorSize)) { + format.UpgradeFrom(formatCandidate.Format); + isOk = true; + } } - } - if (isOk) { - outInfo.Version = format.Version; - outInfo.DiskSize = format.DiskSize; - outInfo.SectorSizeBytes = format.SectorSize; - outInfo.UserAccessibleChunkSizeBytes = format.GetUserAccessibleChunkSize(); - outInfo.DiskGuid = format.Guid; - format.FormatText[sizeof(format.FormatText) - 1] = 0; - outInfo.TextMessage = format.FormatText; - outInfo.RawChunkSizeBytes = format.ChunkSize; - outInfo.SysLogSectorCount = format.SysLogSectorCount; - outInfo.SystemChunkCount = format.SystemChunkCount; - outInfo.Timestamp = TInstant::MicroSeconds(format.TimestampUs); - outInfo.FormatFlags = format.FormatFlagsToString(format.FormatFlags); - - ui32 sysLogSize = format.SectorSize * format.SysLogSectorCount * 3; - ui32 formatBytes = NPDisk::FormatSectorSize * NPDisk::ReplicationFactor; - ui32 sysLogOffsetSectors = (formatBytes + format.SectorSize - 1) / format.SectorSize; - ui32 sysLogOffset = sysLogOffsetSectors * format.SectorSize; - - NPDisk::TAlignedData sysLogRaw(sysLogSize); - NPDisk::TBuffer::TPtr buffer(bufferPool->Pop()); - const ui32 bufferSize = AlignDown(bufferPool->GetBufferSize(), format.SectorSize); - const ui32 sysLogRawParts = (sysLogSize + bufferSize - 1) / bufferSize; - for (ui32 i = 0; i < sysLogRawParts; i++) { - const ui32 sysLogPartSize = Min(bufferSize, sysLogSize - i * bufferSize); - Y_VERIFY(buffer->Size() >= sysLogPartSize); - blockDevice->PreadSync(buffer->Data(), sysLogPartSize, sysLogOffset + i * bufferSize, - NPDisk::TReqId(NPDisk::TReqId::ReadSysLogData, 0), {}); - memcpy(sysLogRaw.Get() + i * bufferSize, buffer->Data(), sysLogPartSize); - } + if (isOk) { + outInfo.Version = format.Version; + outInfo.DiskSize = format.DiskSize; + outInfo.SectorSizeBytes = format.SectorSize; + outInfo.UserAccessibleChunkSizeBytes = format.GetUserAccessibleChunkSize(); + outInfo.DiskGuid = format.Guid; + format.FormatText[sizeof(format.FormatText) - 1] = 0; + outInfo.TextMessage = format.FormatText; + outInfo.RawChunkSizeBytes = format.ChunkSize; + outInfo.SysLogSectorCount = format.SysLogSectorCount; + outInfo.SystemChunkCount = format.SystemChunkCount; + outInfo.Timestamp = TInstant::MicroSeconds(format.TimestampUs); + outInfo.FormatFlags = format.FormatFlagsToString(format.FormatFlags); + + ui32 sysLogSize = format.SectorSize * format.SysLogSectorCount * 3; + ui32 formatBytes = NPDisk::FormatSectorSize * NPDisk::ReplicationFactor; + ui32 sysLogOffsetSectors = (formatBytes + format.SectorSize - 1) / format.SectorSize; + ui32 sysLogOffset = sysLogOffsetSectors * format.SectorSize; + + NPDisk::TAlignedData sysLogRaw(sysLogSize); + NPDisk::TBuffer::TPtr buffer(bufferPool->Pop()); + const ui32 bufferSize = AlignDown(bufferPool->GetBufferSize(), format.SectorSize); + const ui32 sysLogRawParts = (sysLogSize + bufferSize - 1) / bufferSize; + for (ui32 i = 0; i < sysLogRawParts; i++) { + const ui32 sysLogPartSize = Min(bufferSize, sysLogSize - i * bufferSize); + Y_VERIFY(buffer->Size() >= sysLogPartSize); + blockDevice->PreadSync(buffer->Data(), sysLogPartSize, sysLogOffset + i * bufferSize, + NPDisk::TReqId(NPDisk::TReqId::ReadSysLogData, 0), {}); + memcpy(sysLogRaw.Get() + i * bufferSize, buffer->Data(), sysLogPartSize); + } - outInfo.SectorInfo.clear(); - for (ui32 idx = 0; idx < format.SysLogSectorCount * 3; ++idx) { - ui64 logSectorOffset = (ui64)idx * (ui64)format.SectorSize; - ui8 *sector = sysLogRaw.Get() + logSectorOffset; - NPDisk::TDataSectorFooter *logFooter = (NPDisk::TDataSectorFooter*) - (sector + format.SectorSize - sizeof(NPDisk::TDataSectorFooter)); - - ui64 sectorOffset = sysLogOffset + (ui64)((idx / 3) * 3) * (ui64)format.SectorSize; - bool isCrcOk = NPDisk::TPDiskHashCalculator(KIKIMR_PDISK_ENABLE_T1HA_HASH_WRITING).CheckSectorHash( - sectorOffset, format.MagicSysLogChunk, sector, format.SectorSize, logFooter->Hash); - outInfo.SectorInfo.push_back(TPDiskInfo::TSectorInfo(logFooter->Nonce, logFooter->Version, isCrcOk)); - } + outInfo.SectorInfo.clear(); + for (ui32 idx = 0; idx < format.SysLogSectorCount * 3; ++idx) { + ui64 logSectorOffset = (ui64)idx * (ui64)format.SectorSize; + ui8 *sector = sysLogRaw.Get() + logSectorOffset; + NPDisk::TDataSectorFooter *logFooter = (NPDisk::TDataSectorFooter*) + (sector + format.SectorSize - sizeof(NPDisk::TDataSectorFooter)); + + ui64 sectorOffset = sysLogOffset + (ui64)((idx / 3) * 3) * (ui64)format.SectorSize; + bool isCrcOk = NPDisk::TPDiskHashCalculator(KIKIMR_PDISK_ENABLE_T1HA_HASH_WRITING).CheckSectorHash( + sectorOffset, format.MagicSysLogChunk, sector, format.SectorSize, logFooter->Hash); + outInfo.SectorInfo.push_back(TPDiskInfo::TSectorInfo(logFooter->Nonce, logFooter->Version, isCrcOk)); + } - return true; + return true; + } } TStringStream str; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h index 89b8f6418e9..47021792407 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h @@ -46,7 +46,7 @@ void FormatPDisk(TString path, ui64 diskSizeBytes, ui32 sectorSizeBytes, ui32 us const bool isErasureEncodeUserLog = false, const bool trimEntireDevice = false, TIntrusivePtr<NPDisk::TSectorMap> sectorMap = nullptr); -bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TKey &mainKey, TPDiskInfo &outInfo, +bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TMainKey &mainKey, TPDiskInfo &outInfo, const bool doLock = false, TIntrusivePtr<NPDisk::TSectorMap> sectorMap = nullptr); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp index 4266d75266d..e89c9151b4b 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp @@ -674,7 +674,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { NKikimrProto::OK); round = evInitRes->PDiskParams->OwnerRound + 1; - testCtx.MainKey += 123; + testCtx.MainKey[0] += 123; testCtx.UpdateConfigRecreatePDisk(testCtx.GetPDiskConfig()); evInitRes = testCtx.TestResponce<NPDisk::TEvYardInitResult>( @@ -684,5 +684,93 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { new NPDisk::TEvCheckSpace(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound), NKikimrProto::CORRUPTED); } + + Y_UNIT_TEST(ChangePDiskKey) { + const TString data = PrepareData(4096); + + TActorTestContext testCtx(false); + + TVDiskMock mock(&testCtx); + mock.InitFull(); + + mock.ReserveChunk(); + const ui32 chunk = *mock.Chunks[EChunkState::RESERVED].begin(); + + auto readChunk = [&]() { + auto evReadRes = testCtx.TestResponce<NPDisk::TEvChunkReadResult>( + new NPDisk::TEvChunkRead(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound, + chunk, 0, data.size(), 0, nullptr), + NKikimrProto::OK); + UNIT_ASSERT_VALUES_EQUAL(evReadRes->Data.ToString(), data); + }; + + TString dataCopy = data; + testCtx.TestResponce<NPDisk::TEvChunkWriteResult>(new NPDisk::TEvChunkWrite(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound, + chunk, 0, new NPDisk::TEvChunkWrite::TStrokaBackedUpParts(dataCopy), nullptr, false, 0), + NKikimrProto::OK); + mock.CommitReservedChunks(); + + readChunk(); + + testCtx.MainKey.push_back(0xFull); + testCtx.RestartPDiskSync(); + mock.InitFull(); + readChunk(); + + testCtx.MainKey = { 0xFull }; + testCtx.RestartPDiskSync(); + mock.InitFull(); + readChunk(); + + testCtx.MainKey = { 0xFull, 0xA }; + testCtx.RestartPDiskSync(); + mock.InitFull(); + readChunk(); + + testCtx.MainKey = { 0xFull, 0xA, 0xB, 0xC }; + testCtx.RestartPDiskSync(); + mock.InitFull(); + readChunk(); + + testCtx.MainKey = { 0xC }; + testCtx.RestartPDiskSync(); + mock.InitFull(); + readChunk(); + } + + + Y_UNIT_TEST(WrongPDiskKey) { + const TString data = PrepareData(4096); + + TActorTestContext testCtx(false); + + TVDiskMock mock(&testCtx); + mock.InitFull(); + + mock.ReserveChunk(); + const ui32 chunk = *mock.Chunks[EChunkState::RESERVED].begin(); + + TString dataCopy = data; + testCtx.TestResponce<NPDisk::TEvChunkWriteResult>(new NPDisk::TEvChunkWrite(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound, + chunk, 0, new NPDisk::TEvChunkWrite::TStrokaBackedUpParts(dataCopy), nullptr, false, 0), + NKikimrProto::OK); + mock.CommitReservedChunks(); + testCtx.TestResponce<NPDisk::TEvCheckSpaceResult>( + new NPDisk::TEvCheckSpace(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound), + NKikimrProto::OK); + testCtx.TestResponce<NPDisk::TEvChunkReadResult>( + new NPDisk::TEvChunkRead(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound, + chunk, 0, data.size(), 0, nullptr), + NKikimrProto::OK); + + testCtx.MainKey = { 0xABCDEF }; + testCtx.TestResponce<NPDisk::TEvYardControlResult>( + new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStop, nullptr), + NKikimrProto::OK); + + testCtx.TestResponce<NPDisk::TEvYardControlResult>( + new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStart, (void*)(&testCtx.MainKey)), + NKikimrProto::CORRUPTED); + } } } // namespace NKikimr diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h index 6cf9ceb49bf..f6cf4482d01 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h @@ -23,7 +23,7 @@ private: public: TActorId Sender; - NPDisk::TKey MainKey = NPDisk::YdbDefaultPDiskSequence; + NPDisk::TMainKey MainKey = { NPDisk::YdbDefaultPDiskSequence }; TTestContext TestCtx{false, /*use sector map*/ true}; TIntrusivePtr<TPDiskConfig> DefaultPDiskConfig(bool isBad) { @@ -97,7 +97,7 @@ public: if (!PDisk && !UsePDiskMock) { // To be sure that pdisk actor is in StateOnline TestResponce<NPDisk::TEvYardControlResult>( - new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStart, &MainKey), + new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStart, (void*)(&MainKey)), NKikimrProto::OK); const auto evControlRes = TestResponce<NPDisk::TEvYardControlResult>( diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp index a63d37d5c50..ee72a048938 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp @@ -87,8 +87,9 @@ void Run(TVector<IActor*> tests, TTestRunConfig runCfg) { pDiskConfig->EnableSectorEncryption = !pDiskConfig->SectorMap; pDiskConfig->UseT1ha0HashInFooter = runCfg.UseT1ha0Hasher; + NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence}; TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), - NPDisk::YdbDefaultPDiskSequence, mainCounters), TMailboxType::Revolving, 0); + mainKey, mainCounters), TMailboxType::Revolving, 0); setup1->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(pDiskId, pDiskSetup)); for (ui32 i = 0; i < runCfg.Instances; ++i) { diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp index ba7353ad62e..1f98f7a952f 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp @@ -534,7 +534,8 @@ YARD_UNIT_TEST(TestDamagedFirstRecordToKeep) { MakeDirIfNotExist(databaseDirectory.c_str()); } TPDiskInfo info; - bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap); + const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence}; + bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap); UNIT_ASSERT_VALUES_EQUAL(isOk, true); ui32 dataSize = info.SystemChunkCount * info.RawChunkSizeBytes; @@ -880,7 +881,8 @@ YARD_UNIT_TEST(TestFormatInfo) { FormatPDiskForTest(dataPath, tc.PDiskGuid, chunkSize, 1 << 30, false, tc.SectorMap); TPDiskInfo info; - bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap); + const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence}; + bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap); UNIT_ASSERT_VALUES_EQUAL(isOk, true); UNIT_ASSERT_VALUES_EQUAL(info.TextMessage, "Info"); } @@ -914,7 +916,8 @@ YARD_UNIT_TEST(TestRestartAtNonceJump) { MakeDirIfNotExist(databaseDirectory.c_str()); } TPDiskInfo info; - bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap); + const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence}; + bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap); UNIT_ASSERT_VALUES_EQUAL(isOk, true); // Destroy data in chunks starting at# SystemChunkCount + 1 ui32 dataSize = 8 * chunkSize; @@ -943,7 +946,8 @@ YARD_UNIT_TEST(TestRestartAtChunkEnd) { MakeDirIfNotExist(databaseDirectory.c_str()); } TPDiskInfo info; - bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap); + const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence}; + bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap); UNIT_ASSERT_VALUES_EQUAL(isOk, true); // Destroy data in chunks starting at# SystemChunkCount + 1 ui32 dataSize = 8 * chunkSize; diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h index 0f42392943f..1f7fa20c6e3 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h @@ -46,7 +46,7 @@ struct TEnvironmentSetup { {} void Create(const TActorContext& ctx, ui32 pdiskId, const TIntrusivePtr<TPDiskConfig>& cfg, - const NPDisk::TKey& /*mainKey*/, ui32 poolId, ui32 nodeId) override { + const NPDisk::TMainKey& /*mainKey*/, ui32 poolId, ui32 nodeId) override { const auto key = std::make_pair(nodeId, pdiskId); TIntrusivePtr<TPDiskMockState>& state = Env.PDiskMockStates[key]; if (!state) { diff --git a/ydb/core/blobstorage/ut_group/main.cpp b/ydb/core/blobstorage/ut_group/main.cpp index 0ef59b5479c..d703975dcf1 100644 --- a/ydb/core/blobstorage/ut_group/main.cpp +++ b/ydb/core/blobstorage/ut_group/main.cpp @@ -161,7 +161,7 @@ public: }; TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters; - const NPDisk::TKey MainKey = NPDisk::YdbDefaultPDiskSequence; + const NPDisk::TMainKey MainKey = { NPDisk::YdbDefaultPDiskSequence }; const ui32 NodeCount; const ui32 GroupId = 0; std::vector<TDiskRecord> Disks; diff --git a/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h b/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h index 06b5d5a0ba8..f9e1bcf05bb 100644 --- a/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h +++ b/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h @@ -85,7 +85,7 @@ public: Y_VERIFY(ctx.ExecutorThread.ActorSystem); Y_VERIFY(PDiskConfig); Y_VERIFY(AppData(ctx)); - std::unique_ptr<IActor> pdiskActor(CreatePDisk(PDiskConfig, 1, Counters->GetSubgroup("subsystem", "pdisk"))); + std::unique_ptr<IActor> pdiskActor(CreatePDisk(PDiskConfig, {1}, Counters->GetSubgroup("subsystem", "pdisk"))); const TActorId actorId = ctx.ExecutorThread.ActorSystem->Register(pdiskActor.release(), TMailboxType::Simple, AppData(ctx)->SystemPoolId); PDiskServiceId = MakeBlobStoragePDiskID(ctx.ExecutorThread.ActorSystem->NodeId, PDiskConfig->PDiskId); diff --git a/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp b/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp index ccee189f60b..7d691ba7a1e 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp @@ -158,8 +158,9 @@ void TAllPDisks::ActorSetupCmd(NActors::TActorSystemSetup *setup, ui32 node, TPDiskCategory(deviceType, 0).GetRaw())); pDiskConfig->GetDriveDataSwitch = NKikimrBlobStorage::TPDiskConfig::DoNotTouch; pDiskConfig->WriteCacheSwitch = NKikimrBlobStorage::TPDiskConfig::DoNotTouch; + const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence}; TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), - NPDisk::YdbDefaultPDiskSequence, counters), TMailboxType::Revolving, 0); + mainKey, counters), TMailboxType::Revolving, 0); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(inst.PDiskActorID, pDiskSetup)); } } diff --git a/ydb/core/driver_lib/base_utils/base_utils.h b/ydb/core/driver_lib/base_utils/base_utils.h index 4809f2824da..3dfd02853eb 100644 --- a/ydb/core/driver_lib/base_utils/base_utils.h +++ b/ydb/core/driver_lib/base_utils/base_utils.h @@ -14,6 +14,8 @@ #include <library/cpp/string_utils/parse_size/parse_size.h> #include <library/cpp/svnversion/svnversion.h> +#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h> + namespace NKikimr { struct TCmdFormatConfig { @@ -22,7 +24,7 @@ struct TCmdFormatConfig { NSize::TSize ChunkSize; NSize::TSize SectorSize; ui64 Guid; - ui64 MainKey; + NPDisk::TMainKey MainKey; TString TextMessage; TCmdFormatConfig(); @@ -32,7 +34,8 @@ struct TCmdFormatConfig { struct TCmdFormatInfoConfig { TString Path; - ui64 MainKey; + TVector<NPDisk::TKey> MainKeyTmp; // required for .AppendTo() + NPDisk::TMainKey MainKey; bool IsVerbose; TCmdFormatInfoConfig(); diff --git a/ydb/core/driver_lib/base_utils/format_info.cpp b/ydb/core/driver_lib/base_utils/format_info.cpp index e5d53a7b803..03498da3756 100644 --- a/ydb/core/driver_lib/base_utils/format_info.cpp +++ b/ydb/core/driver_lib/base_utils/format_info.cpp @@ -58,7 +58,7 @@ int MainFormatInfo(const TCommandConfig &cmdConf, int argc, char** argv) { } TCmdFormatInfoConfig::TCmdFormatInfoConfig() - : MainKey(0) + : MainKey({}) , IsVerbose(false) {} @@ -69,9 +69,9 @@ void TCmdFormatInfoConfig::Parse(int argc, char **argv) { opts.AddLongOption('p', "pdisk-path", "path to pdisk to read format info").RequiredArgument("PATH").Required() .StoreResult(&Path); opts.AddLongOption('k', "main-key", "encryption main-key to use while reading").RequiredArgument("NUM") - .Optional().StoreResult(&MainKey); // TODO: make required + .Optional().AppendTo(&MainKeyTmp); // TODO: make required opts.AddLongOption("master-key", "obsolete: use main-key").RequiredArgument("NUM") - .Optional().StoreResult(&MainKey); // TODO: remove after migration + .Optional().AppendTo(&MainKeyTmp); // TODO: remove after migration opts.AddLongOption('v', "verbose", "output detailed information for debugging").Optional().NoArgument() .SetFlag(&IsVerbose); @@ -85,6 +85,11 @@ void TCmdFormatInfoConfig::Parse(int argc, char **argv) { bool hasKOption = res.FindCharOptParseResult('k'); if (!hasMainOption && !hasMasterOption && !hasKOption) ythrow yexception() << "missing main-key param"; + + MainKey = {}; + for (auto& key : MainKeyTmp) { + MainKey.push_back(key); + } } } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp index a9be370d37f..ebc4a18930c 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp @@ -14,20 +14,21 @@ public: bool IsVerbose; bool LockDevice; - ui64 MainKey; + TVector<NPDisk::TKey> MainKeyTmp; // required for .AppendTo() + NPDisk::TMainKey MainKey; TString Path; virtual void Config(TConfig& config) override { TClientCommand::Config(config); IsVerbose = false; LockDevice = false; - MainKey = 0; + MainKeyTmp = {}; config.SetFreeArgsNum(1); SetFreeArgTitle(0, "<PATH>", "Disk path"); config.Opts->AddLongOption('k', "main-key", "encryption main-key to use while reading").RequiredArgument("NUM") - .Optional().StoreResult(&MainKey); // TODO: make required + .Optional().AppendTo(&MainKeyTmp); // TODO: make required config.Opts->AddLongOption("master-key", "obsolete: use main-key").RequiredArgument("NUM") - .Optional().StoreResult(&MainKey); // TODO: remove after migration + .Optional().AppendTo(&MainKeyTmp); // TODO: remove after migration config.Opts->AddLongOption('v', "verbose", "output detailed information for debugging").Optional().NoArgument() .SetFlag(&IsVerbose); config.Opts->AddLongOption('l', "lock", "lock device before reading disk info").Optional().NoArgument() @@ -43,6 +44,11 @@ public: bool hasKOption = config.ParseResult->FindCharOptParseResult('k'); if (!hasMainOption && !hasMasterOption && !hasKOption) ythrow yexception() << "missing main-key param"; + + MainKey = {}; + for (auto& key : MainKeyTmp) { + MainKey.push_back(key); + } } virtual int Run(TConfig&) override { @@ -108,13 +114,13 @@ public: NSize::TSize ChunkSize; NSize::TSize SectorSize; ui64 Guid; - ui64 MainKey; + TVector<NPDisk::TKey> MainKeyTmp; TString TextMessage; bool IsErasureEncode; virtual void Config(TConfig& config) override { TClientCommand::Config(config); - MainKey = 0; + MainKey = {}; DiskSize = 0; ChunkSize = 128 << 20; SectorSize = 4 << 10; @@ -136,9 +142,9 @@ public: .StoreResult(&Guid); config.Opts->AddLongOption('k', "main-key", "encryption main-key to set while formatting.\n" "Make sure you use the same master key when you format your pdisks and when you run kikimr.") - .RequiredArgument("NUM").Optional().StoreResult(&MainKey); + .RequiredArgument("NUM").Optional().AppendTo(&MainKeyTmp); config.Opts->AddLongOption("master-key", "obsolete: user main-key") - .RequiredArgument("NUM").Optional().StoreResult(&MainKey); + .RequiredArgument("NUM").Optional().AppendTo(&MainKeyTmp); config.Opts->AddLongOption('t', "text-message", "text message to store in format sector (up to 4000 characters long)") .OptionalArgument("STR").Optional().StoreResult(&TextMessage); config.Opts->AddLongOption('e', "erasure-encode", "erasure-encode data to recover from single-sector failures") @@ -153,6 +159,7 @@ public: NPDisk::TKey ChunkKey; NPDisk::TKey LogKey; NPDisk::TKey SysLogKey; + NPDisk::TMainKey MainKey; virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); @@ -165,10 +172,14 @@ public: bool hasKOption = config.ParseResult->FindCharOptParseResult('k'); if (!hasMainOption && !hasMasterOption && !hasKOption) ythrow yexception() << "missing main-key param"; + + for (auto& key : MainKeyTmp) { + MainKey.push_back(key); + } } virtual int Run(TConfig&) override { - FormatPDisk(Path, DiskSize, SectorSize, ChunkSize, Guid, ChunkKey, LogKey, SysLogKey, MainKey, TextMessage, + FormatPDisk(Path, DiskSize, SectorSize, ChunkSize, Guid, ChunkKey, LogKey, SysLogKey, MainKey.back(), TextMessage, IsErasureEncode); return 0; } diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index fc3ab1e96eb..865f975bef0 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -76,6 +76,7 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("alloc-file", "Allocator config file").OptionalArgument("PATH"); opts.AddLongOption("yql-file", "Yql Analytics config file").OptionalArgument("PATH"); opts.AddLongOption("yq-file", "Yandex Query config file").OptionalArgument("PATH"); + opts.AddLongOption("pdisk-key-file", "pdisk encryption key config file").OptionalArgument("PATH"); } void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResult& res) { diff --git a/ydb/core/testlib/basics/helpers.cpp b/ydb/core/testlib/basics/helpers.cpp index 259e4f9da94..ccc020812ce 100644 --- a/ydb/core/testlib/basics/helpers.cpp +++ b/ydb/core/testlib/basics/helpers.cpp @@ -67,7 +67,7 @@ namespace NKikimr { }; void TStrandedPDiskServiceFactory::Create(const TActorContext &ctx, ui32 pDiskID, - const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) + const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) { Y_UNUSED(ctx); Y_VERIFY(!Runtime.IsRealThreads()); diff --git a/ydb/core/testlib/basics/helpers.h b/ydb/core/testlib/basics/helpers.h index c71e7d84f69..9658b837149 100644 --- a/ydb/core/testlib/basics/helpers.h +++ b/ydb/core/testlib/basics/helpers.h @@ -68,7 +68,7 @@ namespace NFake { {} void Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg, - const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) override; + const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) override; virtual ~TStrandedPDiskServiceFactory() {} |