aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorserg-belyakov <serg-belyakov@yandex-team.com>2022-09-19 11:25:40 +0300
committerserg-belyakov <serg-belyakov@yandex-team.com>2022-09-19 11:25:40 +0300
commitc261c7112a1857f7c093dd87cfd07a93cd219822 (patch)
tree8cffe1c58a2b6f522863997c963dc56a78dfbf05
parentb8ba0035bc558ad4ec20e5e24447a5f20a7ee221 (diff)
downloadydb-c261c7112a1857f7c093dd87cfd07a93cd219822.tar.gz
Add multiple PDisk main keys,
Add pdisk-key option to CLI Fix key obtaining Key change now works Change pdisk main key type to TMainKey (typedefed as TStackVec<NPDisk::TKey, 2>)
-rw-r--r--ydb/core/base/blobstorage.h3
-rw-r--r--ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp6
-rw-r--r--ydb/core/blobstorage/base/blobstorage_events.h4
-rw-r--r--ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp3
-rw-r--r--ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp6
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden.h26
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_impl.cpp34
-rw-r--r--ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp13
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk.h3
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp168
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h29
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h6
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h10
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h1
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h4
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp141
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h3
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp1
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h26
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h2
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp128
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h2
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp90
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h4
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp3
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp12
-rw-r--r--ydb/core/blobstorage/ut_blobstorage/lib/env.h2
-rw-r--r--ydb/core/blobstorage/ut_group/main.cpp2
-rw-r--r--ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h2
-rw-r--r--ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp3
-rw-r--r--ydb/core/driver_lib/base_utils/base_utils.h7
-rw-r--r--ydb/core/driver_lib/base_utils/format_info.cpp11
-rw-r--r--ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp29
-rw-r--r--ydb/core/driver_lib/run/config_parser.cpp1
-rw-r--r--ydb/core/testlib/basics/helpers.cpp2
-rw-r--r--ydb/core/testlib/basics/helpers.h2
36 files changed, 549 insertions, 240 deletions
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
index d8bc5b3529c..a982ae15bd9 100644
--- a/ydb/core/base/blobstorage.h
+++ b/ydb/core/base/blobstorage.h
@@ -687,11 +687,12 @@ struct TEvBlobStorage {
EvNonrestoredCorruptedBlobNotify,
EvHugeLockChunks,
EvHugeStat,
- EvForwardToSkeleton,
+ EvForwardToSkeleton, // 268 636 300
EvHugeUnlockChunks,
EvVDiskStatRequest,
EvGetLogoBlobRequest,
EvChunkForget,
+ EvFormatReencryptionFinish,
EvYardInitResult = EvPut + 9 * 512, /// 268 636 672
EvLogResult,
diff --git a/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp b/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp
index 69a436bdeaf..c74015e50ac 100644
--- a/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp
+++ b/ydb/core/blobstorage/backpressure/queue_backpressure_client_ut.cpp
@@ -129,6 +129,7 @@ class TQueueTestRuntime {
ui32 ChunkSize;
ui64 PDiskGuid;
NPDisk::TKey PDiskKey;
+ NPDisk::TMainKey MainKey;
TActorId PDiskId;
std::unique_ptr<TAppData> AppData;
TVDiskID VDiskId;
@@ -160,7 +161,8 @@ public:
DiskSize = SectorMap->DeviceSize;
PDiskGuid = 1;
PDiskKey = 1;
- FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, PDiskKey, "queue_test",
+ MainKey = {1};
+ FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, MainKey.back(), "queue_test",
false, false, SectorMap);
PDiskId = MakeBlobStoragePDiskID(1, 1);
@@ -170,7 +172,7 @@ public:
pDiskConfig->WriteCacheSwitch = NKikimrBlobStorage::TPDiskConfig::DoNotTouch;
pDiskConfig->SectorMap = SectorMap;
pDiskConfig->EnableSectorEncryption = !pDiskConfig->SectorMap;
- TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), PDiskKey, Counters), TMailboxType::Revolving, 0);
+ TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), MainKey, Counters), TMailboxType::Revolving, 0);
setup->LocalServices.emplace_back(PDiskId, pDiskSetup);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/core/blobstorage/base/blobstorage_events.h b/ydb/core/blobstorage/base/blobstorage_events.h
index 7e7fa781eed..b653832457e 100644
--- a/ydb/core/blobstorage/base/blobstorage_events.h
+++ b/ydb/core/blobstorage/base/blobstorage_events.h
@@ -437,10 +437,10 @@ namespace NKikimr {
struct TEvBlobStorage::TEvRestartPDisk : TEventLocal<TEvRestartPDisk, EvRestartPDisk> {
const ui32 PDiskId;
- NPDisk::TKey MainKey;
+ const NPDisk::TMainKey MainKey;
TIntrusivePtr<TPDiskConfig> Config;
- TEvRestartPDisk(const ui32& pdiskId, const NPDisk::TKey& mainKey, const TIntrusivePtr<TPDiskConfig>& config)
+ TEvRestartPDisk(const ui32& pdiskId, const NPDisk::TMainKey& mainKey, const TIntrusivePtr<TPDiskConfig>& config)
: PDiskId(pdiskId)
, MainKey(mainKey)
, Config(config)
diff --git a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp
index 07f5948f3ae..1d881f19f01 100644
--- a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp
+++ b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp
@@ -4227,8 +4227,9 @@ public:
pDiskConfig->SectorMap = SectorMapByPath[filePath];
pDiskConfig->EnableSectorEncryption = !pDiskConfig->SectorMap;
+ NPDisk::TMainKey mainKeys = { mainKey };
TActorSetupCmd pDiskSetup(
- CreatePDisk(pDiskConfig.Get(), mainKey, counters),
+ CreatePDisk(pDiskConfig.Get(), mainKeys, counters),
TMailboxType::Revolving, 0);
setup2->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(pdiskId, pDiskSetup));
diff --git a/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp b/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp
index ae065199ff5..a4f8b3e15f5 100644
--- a/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp
+++ b/ydb/core/blobstorage/incrhuge/ut/incrhuge_basic_ut.cpp
@@ -29,6 +29,7 @@ public:
ui64 DiskSize;
ui64 PDiskGuid;
ui64 PDiskKey;
+ NPDisk::TMainKey MainKey;
TActorId PDiskId;
TActorId KeeperId;
std::unique_ptr<TActorSystem> ActorSystem;
@@ -57,13 +58,14 @@ public:
DiskSize = (ui64)ChunkSize * numChunks;
PDiskGuid = Now().GetValue();
PDiskKey = 1;
- FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, PDiskKey, "incrhuge");
+ MainKey = {1};
+ FormatPDisk(Path, DiskSize, 4096, ChunkSize, PDiskGuid, PDiskKey, PDiskKey, PDiskKey, MainKey.back(), "incrhuge");
}
PDiskId = MakeBlobStoragePDiskID(1, 1);
ui64 pDiskCategory = 0;
TIntrusivePtr<TPDiskConfig> pDiskConfig = new TPDiskConfig(Path, PDiskGuid, 1, pDiskCategory);
- TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), PDiskKey, Counters), TMailboxType::Revolving, 0);
+ TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(), MainKey, Counters), TMailboxType::Revolving, 0);
setup->LocalServices.emplace_back(PDiskId, pDiskSetup);
TActorId pdiskActorId;
diff --git a/ydb/core/blobstorage/nodewarden/node_warden.h b/ydb/core/blobstorage/nodewarden/node_warden.h
index 9ecc35bd3c1..fb1cfe370c5 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden.h
+++ b/ydb/core/blobstorage/nodewarden/node_warden.h
@@ -28,7 +28,7 @@ namespace NKikimr {
std::unique_ptr<ICacheAccessor> CacheAccessor;
TEncryptionKey TenantKey;
TEncryptionKey StaticKey;
- TEncryptionKey PDiskKey;
+ TVector<TEncryptionKey> PDiskKey;
bool CachePDisks = false;
bool CacheVDisks = false;
bool EnableVDiskCooldownTimeout = false;
@@ -42,15 +42,19 @@ namespace NKikimr {
, AllDriveModels(new NPDisk::TDriveModelDb)
{}
- NPDisk::TKey CreatePDiskKey() const {
- if (PDiskKey) {
- const ui8 *key;
- ui32 keySize;
- PDiskKey.Key.GetKeyBytes(&key, &keySize);
- return *(ui64*)key;
- } else {
- return NPDisk::YdbDefaultPDiskSequence;
- }
+ NPDisk::TMainKey CreatePDiskKey() const {
+ if (PDiskKey.size() > 0) {
+ NPDisk::TMainKey mainKey;
+ for (ui32 i = 0; i < PDiskKey.size(); ++i) {
+ const ui8 *key;
+ ui32 keySize;
+ PDiskKey[i].Key.GetKeyBytes(&key, &keySize);
+ mainKey.push_back(*(ui64*)key);
+ }
+ return mainKey;
+ } else {
+ return { NPDisk::YdbDefaultPDiskSequence };
+ }
}
bool IsCacheEnabled() const {
@@ -62,7 +66,7 @@ namespace NKikimr {
bool ObtainTenantKey(TEncryptionKey *key, const NKikimrProto::TKeyConfig& keyConfig);
bool ObtainStaticKey(TEncryptionKey *key);
- bool ObtainPDiskKey(TEncryptionKey *key, const NKikimrProto::TKeyConfig& keyConfig);
+ bool ObtainPDiskKey(TVector<TEncryptionKey> *key, const NKikimrProto::TKeyConfig& keyConfig);
std::unique_ptr<ICacheAccessor> CreateFileCacheAccessor(const TString& templ, const std::unordered_map<char, TString>& vars);
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
index 02586ae4506..4457106f62c 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
@@ -489,14 +489,38 @@ bool NKikimr::ObtainTenantKey(TEncryptionKey *key, const NKikimrProto::TKeyConfi
}
}
-bool NKikimr::ObtainPDiskKey(TEncryptionKey *key, const NKikimrProto::TKeyConfig& keyConfig) {
- if (keyConfig.KeysSize()) {
- auto &record = keyConfig.GetKeys(0);
- return ObtainKey(key, record);
- } else {
+bool NKikimr::ObtainPDiskKey(TVector<TEncryptionKey> *keys, const NKikimrProto::TKeyConfig& keyConfig) {
+ ui32 keysSize = keyConfig.KeysSize();
+ if (!keysSize) {
Cerr << "No Keys in PDiskKeyConfig! Encrypted pdisks will not start" << Endl;
return false;
}
+
+ keys->resize(keysSize);
+ for (ui32 i = 0; i < keysSize; ++i) {
+ auto &record = keyConfig.GetKeys(i);
+ if (record.GetId() == "0" && record.GetContainerPath() == "") {
+ // use default pdisk key
+ (*keys)[i].Id = "0";
+ (*keys)[i].Version = record.GetVersion();
+
+ ui8 *keyBytes = 0;
+ ui32 keySize = 0;
+ (*keys)[i].Key.MutableKeyBytes(&keyBytes, &keySize);
+
+ ui64* p = (ui64*)keyBytes;
+ p[0] = NPDisk::YdbDefaultPDiskSequence;
+ } else {
+ if (!ObtainKey(&(*keys)[i], record)) {
+ return false;
+ }
+ }
+ }
+
+ std::sort(keys->begin(), keys->end(), [&](const TEncryptionKey& l, const TEncryptionKey& r) {
+ return l.Version < r.Version;
+ });
+ return true;
}
diff --git a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp
index edabea0c153..07a0b1e3ef7 100644
--- a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp
+++ b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp
@@ -107,12 +107,15 @@ namespace NKikimr::NStorage {
pdiskConfig->EnableSectorEncryption = !pdiskConfig->SectorMap;
}
- NPDisk::TKey pdiskKey = Cfg->CreatePDiskKey();
- THashCalculator hasher;
+ NPDisk::TMainKey pdiskKey = Cfg->CreatePDiskKey();
TString keyPrintSalt = "@N2#_lW19)2-31!iifI@n1178349617";
- hasher.Hash(keyPrintSalt.Detach(), keyPrintSalt.Size());
- hasher.Hash(&pdiskKey, sizeof(pdiskKey));
- pdiskConfig->HashedMainKey = TStringBuilder() << Hex(hasher.GetHashResult(), HF_ADDX);
+ pdiskConfig->HashedMainKey.resize(pdiskKey.size());
+ for (ui32 i = 0; i < pdiskKey.size(); ++i) {
+ THashCalculator hasher;
+ hasher.Hash(keyPrintSalt.Detach(), keyPrintSalt.Size());
+ hasher.Hash(&pdiskKey[i], sizeof(pdiskKey[i]));
+ pdiskConfig->HashedMainKey[i] = TStringBuilder() << Hex(hasher.GetHashResult(), HF_ADDX);
+ }
pdiskConfig->Initialize();
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
index 765a05d4782..62efca4a615 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
@@ -16,7 +16,7 @@
namespace NKikimr {
-IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey,
+IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey,
const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters);
namespace NPDisk {
@@ -124,7 +124,6 @@ public:
}
};
-
////////////////////////////////////////////////////////////////////////////
// INIT
////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp
index 220748cd181..b3d89908b25 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp
@@ -67,7 +67,7 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
TString StateErrorReason;
TIntrusivePtr<TPDiskConfig> Cfg;
- TKey MainKey;
+ NPDisk::TMainKey MainKey;
TList<TInitQueueItem> InitQueue;
const TIntrusivePtr<::NMonitoring::TDynamicCounters> PDiskCounters;
TIntrusivePtr<TPDisk> PDisk;
@@ -195,7 +195,7 @@ public:
return NKikimrServices::TActivity::PDISK_ACTOR;
}
- TPDiskActor(const TIntrusivePtr<TPDiskConfig>& cfg, const NPDisk::TKey &mainKey,
+ TPDiskActor(const TIntrusivePtr<TPDiskConfig>& cfg, const NPDisk::TMainKey& mainKey,
const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters)
: Cfg(cfg)
, MainKey(mainKey)
@@ -203,10 +203,11 @@ public:
->GetSubgroup("pdisk", Sprintf("%09" PRIu32, (ui32)cfg->PDiskId))
->GetSubgroup("media", to_lower(cfg->PDiskCategory.TypeStrShort())))
{
+ Y_VERIFY(!MainKey.empty());
}
~TPDiskActor() {
- SecureWipeBuffer((ui8*)&MainKey, sizeof(MainKey));
+ SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size());
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -362,7 +363,7 @@ public:
try {
FormatPDisk(cfg->GetDevicePath(), 0, cfg->SectorSize, cfg->ChunkSize,
- cfg->PDiskGuid, chunkKey, logKey, sysLogKey, actor->MainKey, TString(), false,
+ cfg->PDiskGuid, chunkKey, logKey, sysLogKey, actor->MainKey.back(), TString(), false,
cfg->FeatureFlags.GetTrimEntireDeviceOnStartup(), cfg->SectorMap);
actorSystem->Send(pDiskActor, new TEvPDiskFormattingFinished(true, ""));
} catch (yexception ex) {
@@ -375,7 +376,7 @@ public:
FormattingThread->Start();
} else {
- SecureWipeBuffer((ui8*)&MainKey, sizeof(MainKey));
+ SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size());
*PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError;
*PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error;
*PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorPDiskCannotBeInitialised;
@@ -386,55 +387,133 @@ public:
}
TStringStream str;
str << "PDiskId# " << PDisk->PDiskId
- << " Can not be initialized! " << PDisk->ErrorStr
- << " Hash(MainKey)# " << Cfg->HashedMainKey;
+ << " Can not be initialized! " << PDisk->ErrorStr;
+ for (ui32 i = 0; i < Cfg->HashedMainKey.size(); ++i) {
+ str << " Hash(NewMainKey[" << i << "])# " << Cfg->HashedMainKey[i];
+ }
InitError(str.Str());
str << " Config: " << Cfg->ToString();
LOG_CRIT_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str());
}
}
+ void ReencryptDiskFormat(const TDiskFormat& format, const NPDisk::TKey& newMainKey) {
+ IsFormattingNow = true;
+ // Stop PDiskThread but use PDisk object for creation of http pages
+ PDisk->Stop();
+ *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingReencryptingFormat;
+ PDisk->ErrorStr = " Format sectors are encrypted with an old PDisk key, reencryption started";
+ LOG_WARN_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId << PDisk->ErrorStr);
+
+ // Is used to pass parameters into formatting thread, because TThread can pass only void*
+ using TCookieType = std::tuple<TDiskFormat, NPDisk::TKey, TIntrusivePtr<TPDiskConfig>, NActors::TActorSystem*, TActorId>;
+ FormattingThread.Reset(new TThread(
+ [] (void *cookie) -> void* {
+ std::unique_ptr<TCookieType> params(static_cast<TCookieType*>(cookie));
+ TDiskFormat format = std::get<0>(*params);
+ NPDisk::TKey mainKey = std::get<1>(*params);
+ TIntrusivePtr<TPDiskConfig> cfg = std::get<2>(*params);
+ const TIntrusivePtr<::NMonitoring::TDynamicCounters> counters(new ::NMonitoring::TDynamicCounters);
+ NActors::TActorSystem* actorSystem = std::get<3>(*params);
+ TActorId pdiskActor = std::get<4>(*params);
+
+ THolder<NPDisk::TPDisk> pDisk(new NPDisk::TPDisk(cfg, counters));
+
+ pDisk->Initialize(actorSystem, TActorId());
+
+ if (!pDisk->BlockDevice->IsGood()) {
+ ythrow yexception() << "Failed to initialize temporal PDisk for format rewriting, info# " << pDisk->BlockDevice->DebugInfo();
+ }
+
+ try {
+ pDisk->WriteApplyFormatRecord(format, mainKey);
+ actorSystem->Send(pdiskActor, new TEvFormatReencryptionFinish(true, ""));
+ } catch (yexception ex) {
+ LOG_ERROR_S(*actorSystem, NKikimrServices::BS_PDISK, "Reencryption error, what#" << ex.what());
+ actorSystem->Send(pdiskActor, new TEvFormatReencryptionFinish(false, ex.what()));
+ }
+ return nullptr;
+ },
+ new TCookieType(format, newMainKey, PDisk->Cfg, TlsActivationContext->ActorSystem(), SelfId())
+ ));
+ FormattingThread->Start();
+ }
+
+ void InitHandle(TEvFormatReencryptionFinish::TPtr &ev) {
+ FormattingThread->Join();
+ IsFormattingNow = false;
+ if (ev->Get()->Success) {
+ StartPDiskThread();
+ LOG_WARN_S(*TlsActivationContext, NKikimrServices::BS_PDISK,
+ "PDiskId# " << PDisk->PDiskId << " format chunks reencryption finished");
+ } else {
+ PDisk.Reset(new TPDisk(Cfg, PDiskCounters));
+ PDisk->Initialize(TlsActivationContext->ActorSystem(), SelfId());
+ Y_VERIFY(PDisk->PDiskThread.Running());
+
+ *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError;
+ *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error;
+ *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorDiskCannotBeFormated;
+
+ PDisk->ErrorStr = ToString("Format chunks cannot be reencrypted! Reason# ") + ev->Get()->ErrorReason;
+
+ TStringStream str;
+ str << "PDiskId# " << (ui32)PDisk->PDiskId
+ << " Format chunks cannot be reencrypted! Reason# " << ev->Get()->ErrorReason
+ << " Switching to StateError. Config: " << Cfg->ToString();
+ LOG_CRIT_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str());
+ InitError(str.Str());
+ }
+ }
+
void InitHandle(TEvReadFormatResult::TPtr &ev) {
ui8 *formatSectors = ev->Get()->FormatSectors.Get();
ui32 formatSectorsSize = ev->Get()->FormatSectorsSize;
NSan::CheckMemIsInitialized(formatSectors, formatSectorsSize);
- bool isFormatOk = PDisk->ReadChunk0Format(formatSectors, MainKey);
- if (!isFormatOk) {
+ TCheckDiskFormatResult res = PDisk->ReadChunk0Format(formatSectors, MainKey);
+ if (!res.IsFormatPresent) {
*PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingFormatMagicChecking;
- PDisk->ErrorStr = "Format is not Ok, now checking for proper magic sector on disk";
+ PDisk->ErrorStr = "Format chunks are not present on disk or corrupted, now checking for proper magic sector on disk";
CheckMagicSector(formatSectors, formatSectorsSize);
} else {
- SecureWipeBuffer((ui8*)&MainKey, sizeof(MainKey));
- // Format is read OK
- LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId
- << " Successfully read format record# " << PDisk->Format.ToString());
- TString info;
- if (!PDisk->CheckGuid(&info)) {
- *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError;
- *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error;
- *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadDueToGuid;
- PDisk->ErrorStr = TStringBuilder() << "Can't start due to a guid error " << info;
- TStringStream str;
- str << "PDiskId# " << PDisk->PDiskId << PDisk->ErrorStr;
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str());
- InitError(str.Str());
- } else if (!PDisk->CheckFormatComplete()) {
- *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError;
- *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error;
- *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadIncompleteFormat;
- PDisk->ErrorStr = "Can't start due to incomplete format!";
- TStringStream str;
- str << "PDiskId# " << PDisk->PDiskId << " " << PDisk->ErrorStr << " "
- << "Please, do not trun off your server or remove your storage device while formatting. "
- << "We are sure you did this or something even more creative, like killing the formatter.";
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str());
- InitError(str.Str());
+ if (res.IsReencryptionRequired) {
+ // Format reencryption required
+ ReencryptDiskFormat(PDisk->Format, MainKey.back());
+ // We still need main key after restart
+ // SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size());
} else {
- // PDisk GUID is OK and format is complete
- *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialSysLogRead;
- *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingSysLogRead;
- PDisk->Format.InitMagic();
- PDisk->ReadSysLog(SelfId());
+ // Format is read OK
+ SecureWipeBuffer((ui8*)MainKey.data(), sizeof(NPDisk::TKey) * MainKey.size());
+ LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId
+ << " Successfully read format record# " << PDisk->Format.ToString());
+ TString info;
+ if (!PDisk->CheckGuid(&info)) {
+ *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError;
+ *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error;
+ *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadDueToGuid;
+ PDisk->ErrorStr = TStringBuilder() << "Can't start due to a guid error " << info;
+ TStringStream str;
+ str << "PDiskId# " << PDisk->PDiskId << PDisk->ErrorStr;
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str());
+ InitError(str.Str());
+ } else if (!PDisk->CheckFormatComplete()) {
+ *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialFormatReadError;
+ *PDisk->Mon.PDiskBriefState = TPDiskMon::TPDisk::Error;
+ *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::ErrorInitialFormatReadIncompleteFormat;
+ PDisk->ErrorStr = "Can't start due to incomplete format!";
+ TStringStream str;
+ str << "PDiskId# " << PDisk->PDiskId << " " << PDisk->ErrorStr << " "
+ << "Please, do not turn off your server or remove your storage device while formatting. "
+ << "We are sure you did this or something even more creative, like killing the formatter.";
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::BS_PDISK, str.Str());
+ InitError(str.Str());
+ } else {
+ // PDisk GUID is OK and format is complete
+ *PDisk->Mon.PDiskState = NKikimrBlobStorage::TPDiskState::InitialSysLogRead;
+ *PDisk->Mon.PDiskDetailedState = TPDiskMon::TPDisk::BootingSysLogRead;
+ PDisk->Format.InitMagic();
+ PDisk->ReadSysLog(SelfId());
+ }
}
}
}
@@ -638,7 +717,7 @@ public:
switch (evControl.Action) {
case TEvYardControl::PDiskStart:
{
- auto *mainKey = static_cast<const NPDisk::TKey*>(evControl.Cookie);
+ auto *mainKey = static_cast<const NPDisk::TMainKey*>(evControl.Cookie);
Y_VERIFY(mainKey);
MainKey = *mainKey;
StartPDiskThread();
@@ -879,7 +958,7 @@ public:
}
MainKey = ev->Get()->MainKey;
- SecureWipeBuffer((ui8*)&ev->Get()->MainKey, sizeof(ev->Get()->MainKey));
+ SecureWipeBuffer((ui8*)ev->Get()->MainKey.data(), sizeof(NPDisk::TKey) * ev->Get()->MainKey.size());
LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId
<< " Going to restart PDisk since recieved TEvRestartPDisk");
PDisk->Stop();
@@ -1084,6 +1163,7 @@ public:
cFunc(TEvents::TSystem::Wakeup, HandleWakeup);
hFunc(NPDisk::TEvDeviceError, Handle);
hFunc(TEvBlobStorage::TEvRestartPDisk, Handle);
+ hFunc(NPDisk::TEvFormatReencryptionFinish, InitHandle);
)
STRICT_STFUNC(StateOnline,
@@ -1149,13 +1229,13 @@ public:
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// PDisk Creation
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey,
+IActor* CreatePDisk(const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey,
const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) {
return new NPDisk::TPDiskActor(cfg, mainKey, counters);
}
void TRealPDiskServiceFactory::Create(const TActorContext &ctx, ui32 pDiskID,
- const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) {
+ const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) {
TActorId actorId = ctx.ExecutorThread.RegisterActor(
CreatePDisk(cfg, mainKey, AppData(ctx)->Counters), TMailboxType::ReadAsFilled, poolId);
TActorId pDiskServiceId = MakeBlobStoragePDiskID(nodeId, pDiskID);
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h
index 5e861add10d..0d1ca44e052 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h
@@ -335,5 +335,34 @@ public:
}
};
+class TCompletionSequence : public TCompletionAction {
+ TVector<TCompletionAction*> Actions;
+
+public:
+ TCompletionSequence() = default;
+ TCompletionSequence(const TVector<TCompletionAction*>& actions)
+ : Actions(actions)
+ {}
+
+ TCompletionSequence(TVector<TCompletionAction*>&& actions)
+ : Actions(std::move(actions))
+ {}
+
+ TCompletionSequence& operator=(const TCompletionSequence&) = delete;
+ TCompletionSequence& operator=(TCompletionSequence&&) = delete;
+
+ void Exec(TActorSystem *actorSystem) override {
+ for (TCompletionAction* action : Actions) {
+ action->Exec(actorSystem);
+ }
+ }
+
+ void Release(TActorSystem *actorSystem) override {
+ for (TCompletionAction* action : Actions) {
+ action->Release(actorSystem);
+ }
+ }
+};
+
} // NPDisk
} // NKikimr
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h
index 5990a2ee874..94440f6c8ee 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h
@@ -89,7 +89,7 @@ struct TPDiskConfig : public TThrRefBase {
ui64 PDiskGuid; // set only by constructor
ui32 PDiskId; // set only by constructor
TPDiskCategory PDiskCategory; // set only by constructor
- TString HashedMainKey;
+ TStackVec<TString, 2> HashedMainKey;
ui64 StartOwnerRound = 1ull; // set only by warden
TIntrusivePtr<NPDisk::TSectorMap> SectorMap; // set only by warden
@@ -244,7 +244,9 @@ struct TPDiskConfig : public TThrRefBase {
str << " PDiskGuid# " << PDiskGuid << x;
str << " PDiskId# " << PDiskId << x;
str << " PDiskCategory# " << PDiskCategory.ToString() << x;
- str << " HashedMainKey# " << HashedMainKey << x;
+ for (ui32 i = 0; i < HashedMainKey.size(); ++i) {
+ str << " HashedMainKey[" << i << "]# " << HashedMainKey[i] << x;
+ }
str << " StartOwnerRound# " << StartOwnerRound << x;
str << " SectorMap# " << (SectorMap ? "true" : "false") << x;
str << " EnableSectorEncryption # " << EnableSectorEncryption << x;
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h
index 2b1350bff79..c80fd915cd9 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h
@@ -777,6 +777,16 @@ union TDiskFormatSector {
static_assert(sizeof(TDiskFormat) <= FormatSectorSize, "TDiskFormat size too lagre!");
+struct TCheckDiskFormatResult {
+ bool IsFormatPresent;
+ bool IsReencryptionRequired;
+
+ TCheckDiskFormatResult(bool isFormatPresent, bool isReencryptionRequired)
+ : IsFormatPresent(isFormatPresent)
+ , IsReencryptionRequired(isReencryptionRequired)
+ {}
+};
+
} // NPDisk
} // NKikimr
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h
index cc63207fc7f..8b1fb2c8fc7 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h
@@ -38,6 +38,7 @@ namespace NKikimr {
typedef ui32 TStatusFlags;
typedef ui64 TKey;
typedef ui64 THash;
+ typedef TStackVec<TKey, 2> TMainKey;
struct TOwnerToken {
TOwner Owner = 0;
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h
index 6d472eab412..9c2886e2081 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_factory.h
@@ -13,13 +13,13 @@ namespace NKikimr {
class IPDiskServiceFactory : public TThrRefBase {
public:
virtual void Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg,
- const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) = 0;
+ const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) = 0;
};
class TRealPDiskServiceFactory : public IPDiskServiceFactory {
public:
void Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg,
- const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) override;
+ const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) override;
};
} // NKikimr
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
index d7e554675c3..14e876426d8 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
@@ -101,80 +101,85 @@ TString TPDisk::DynamicStateToString(bool isMultiline) {
return str.Str();
}
-bool TPDisk::ReadChunk0Format(ui8* formatSectors, const TKey& mainKey) {
+TCheckDiskFormatResult TPDisk::ReadChunk0Format(ui8* formatSectors, const NPDisk::TMainKey& mainKey) {
TGuard<TMutex> guard(StateMutex);
- TPDiskStreamCypher cypher(true); // Format record is always encrypted
- cypher.SetKey(mainKey);
- Format.SectorSize = FormatSectorSize;
- ui32 lastGoodIdx = (ui32)-1;
- bool isBad[ReplicationFactor];
- bool isBadPresent = false;
-
- for (ui32 i = 0; i < ReplicationFactor; ++i) {
- ui64 sectorOffset = i * FormatSectorSize;
- ui8* formatSector = formatSectors + sectorOffset;
- TDataSectorFooter *footer = (TDataSectorFooter*)
- (formatSector + FormatSectorSize - sizeof(TDataSectorFooter));
-
- cypher.StartMessage(footer->Nonce);
- alignas(16) TDiskFormat diskFormat;
- cypher.Encrypt(&diskFormat, formatSector, sizeof(TDiskFormat));
-
- isBad[i] = !diskFormat.IsHashOk(FormatSectorSize);
- if (!isBad[i]) {
- Format.UpgradeFrom(diskFormat);
- if (Format.IsErasureEncodeUserChunks()) {
- LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId
- << " Read from disk Format has FormatFlagErasureEncodeUserChunks set, "
- << " but current version of PDisk can't work with it"
- << " Format# " << Format.ToString()
- << " Marker# BPD80");
- Y_FAIL_S("PDiskId# " << PDiskId
- << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserChunks set");
- }
- if (Format.IsErasureEncodeUserLog()) {
- LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId
- << " Read from disk Format has FormatFlagErasureEncodeUserLog set, "
- << " but current version of PDisk can't work with it"
- << " Format# " << Format.ToString()
- << " Marker# BPD801");
- Y_FAIL_S("PDiskId# " << PDiskId
- << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserLog set");
- }
- lastGoodIdx = i;
- *Mon.TotalSpaceBytes = Format.DiskSize;
- } else {
- isBadPresent = true;
- }
- }
-
- if (lastGoodIdx < ReplicationFactor) {
- ui64 sectorOffset = lastGoodIdx * FormatSectorSize;
- ui8* formatSector = formatSectors + sectorOffset;
- if (isBadPresent) {
- for (ui32 i = 0; i < ReplicationFactor; ++i) {
- if (isBad[i]) {
- TBuffer* buffer = BufferPool->Pop();
- Y_VERIFY(FormatSectorSize <= buffer->Size());
- memcpy(buffer->Data(), formatSector, FormatSectorSize);
- ui64 targetOffset = i * FormatSectorSize;
- LOG_INFO_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << (ui32)PDiskId
- << " PWriteAsync offset# " << targetOffset
- << " to# " << (targetOffset + FormatSectorSize)
- << " for format restoration"
- << " Marker# BPD46");
-
- REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer->Data(), FormatSectorSize);
- BlockDevice->PwriteAsync(buffer->Data(), FormatSectorSize, targetOffset, buffer,
- TReqId(TReqId::RestoreFormatOnRead, 0), {});
+ Format.SectorSize = FormatSectorSize;
+ ui32 mainKeySize = mainKey.size();
+
+ for (ui32 k = 0; k < mainKeySize; ++k) {
+ TPDiskStreamCypher cypher(true); // Format record is always encrypted
+ cypher.SetKey(mainKey[k]);
+
+ ui32 lastGoodIdx = (ui32)-1;
+ bool isBad[ReplicationFactor];
+ bool isBadPresent = false;
+ for (ui32 i = 0; i < ReplicationFactor; ++i) {
+ ui64 sectorOffset = i * FormatSectorSize;
+ ui8* formatSector = formatSectors + sectorOffset;
+ TDataSectorFooter *footer = (TDataSectorFooter*)
+ (formatSector + FormatSectorSize - sizeof(TDataSectorFooter));
+
+ cypher.StartMessage(footer->Nonce);
+ alignas(16) TDiskFormat diskFormat;
+ cypher.Encrypt(&diskFormat, formatSector, sizeof(TDiskFormat));
+
+ isBad[i] = !diskFormat.IsHashOk(FormatSectorSize);
+ if (!isBad[i]) {
+ Format.UpgradeFrom(diskFormat);
+ if (Format.IsErasureEncodeUserChunks()) {
+ LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId
+ << " Read from disk Format has FormatFlagErasureEncodeUserChunks set, "
+ << " but current version of PDisk can't work with it"
+ << " Format# " << Format.ToString()
+ << " Marker# BPD80");
+ Y_FAIL_S("PDiskId# " << PDiskId
+ << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserChunks set");
+ }
+ if (Format.IsErasureEncodeUserLog()) {
+ LOG_ERROR_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << PDiskId
+ << " Read from disk Format has FormatFlagErasureEncodeUserLog set, "
+ << " but current version of PDisk can't work with it"
+ << " Format# " << Format.ToString()
+ << " Marker# BPD801");
+ Y_FAIL_S("PDiskId# " << PDiskId
+ << "Unable to run PDisk on disk with FormatFlagErasureEncodeUserLog set");
}
+ lastGoodIdx = i;
+ *Mon.TotalSpaceBytes = Format.DiskSize;
+ } else {
+ isBadPresent = true;
+ }
+ }
+ if (lastGoodIdx < ReplicationFactor) {
+ ui64 sectorOffset = lastGoodIdx * FormatSectorSize;
+ ui8* formatSector = formatSectors + sectorOffset;
+ if (k < mainKeySize - 1) { // obsolete key is used
+ return TCheckDiskFormatResult(true, true);
+ } else if (isBadPresent) {
+ for (ui32 i = 0; i < ReplicationFactor; ++i) {
+ if (isBad[i]) {
+ TBuffer* buffer = BufferPool->Pop();
+ Y_VERIFY(FormatSectorSize <= buffer->Size());
+ memcpy(buffer->Data(), formatSector, FormatSectorSize);
+ ui64 targetOffset = i * FormatSectorSize;
+ LOG_INFO_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << (ui32)PDiskId
+ << " PWriteAsync offset# " << targetOffset
+ << " to# " << (targetOffset + FormatSectorSize)
+ << " for format restoration"
+ << " Marker# BPD46");
+
+ REQUEST_VALGRIND_CHECK_MEM_IS_DEFINED(buffer->Data(), FormatSectorSize);
+ BlockDevice->PwriteAsync(buffer->Data(), FormatSectorSize, targetOffset, buffer,
+ TReqId(TReqId::RestoreFormatOnRead, 0), {});
+ }
+ }
+ //BlockDevice->FlushAsync(nullptr);
}
- //BlockDevice->FlushAsync(nullptr);
+ return TCheckDiskFormatResult(true, false);
}
- return true;
}
- return false;
+ return TCheckDiskFormatResult(false, false);
}
bool TPDisk::IsFormatMagicValid(ui8 *magicData8, ui32 magicDataSize) {
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
index beeec00d4d1..c86b0a6be56 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
@@ -181,7 +181,7 @@ public:
// Initialization
TPDisk(const TIntrusivePtr<TPDiskConfig> cfg, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters);
TString DynamicStateToString(bool isMultiline);
- bool ReadChunk0Format(ui8* formatSectors, const TKey& mainKey); // Called by actor
+ TCheckDiskFormatResult ReadChunk0Format(ui8* formatSectors, const TMainKey& mainKey); // Called by actor
bool IsFormatMagicValid(ui8 *magicData, ui32 magicDataSize); // Called by actor
bool CheckGuid(TString *outReason); // Called by actor
bool CheckFormatComplete(); // Called by actor
@@ -190,6 +190,7 @@ public:
void PrintChunksDebugInfo();
TString ProcessReadSysLogResult(ui64 &outWritePosition, ui64 &outLsn, const TEvReadLogResult &readLogResult);
void ReadAndParseMainLog(const TActorId &pDiskActor);
+ void WriteFormatAsync(TDiskFormat format, const TKey &mainKey);
// Called by the log reader on success with the current chunkOwnerMap.
void ProcessChunkOwnerMap(TMap<ui32, TChunkState> &chunkOwnerMap);
void InitLogChunksInfo();
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp
index 0552ca8c88a..4f01fdad8b1 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp
@@ -233,6 +233,7 @@ void TPDisk::ProcessChunk0(const NPDisk::TEvReadLogResult &readLogResult) {
ui32 chunkCount = (ui32)(Format.DiskSize / (ui64)Format.ChunkSize);
Y_VERIFY_DEBUG(ChunkState.size() == 0);
ChunkState = TVector<TChunkState>(chunkCount);
+ Y_VERIFY(ChunkState.size() >= Format.SystemChunkCount);
for (ui32 i = 0; i < Format.SystemChunkCount; ++i) {
ChunkState[i].OwnerId = OwnerSystem;
}
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h
index df2f0885694..e8a38d3eeeb 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_internal_interface.h
@@ -196,5 +196,31 @@ struct TEvDeviceError : public TEventLocal<TEvDeviceError, TEvBlobStorage::EvDev
{}
};
+struct TEvFormatReencryptionFinish : public TEventLocal<TEvFormatReencryptionFinish, TEvBlobStorage::EvFormatReencryptionFinish> {
+ bool Success;
+ TString ErrorReason;
+
+ TEvFormatReencryptionFinish(bool success, TString errorReason)
+ : Success(success)
+ , ErrorReason(errorReason)
+ {}
+
+ TString ToString() const {
+ return ToString(*this);
+ }
+
+ static TString ToString(const TEvFormatReencryptionFinish& record) {
+ TStringStream str;
+ str << "{";
+ str << "EvFormatReencryptionFinished ";
+ str << " Success# " << record.Success;
+ if (record.ErrorReason) {
+ str << " ErrorReason# " << record.ErrorReason;
+ }
+ str << "}";
+ return str.Str();
+ }
+};
+
} // NPDisk
} // NKikimr
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h
index 84fd08d685c..b5b10e63d44 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h
@@ -361,6 +361,7 @@ struct TPDiskMon {
ErrorNoDeviceWithSuchSerial,
ErrorDeviceSerialMismatch,
ErrorFake,
+ BootingReencryptingFormat,
};
static TString StateToStr(i64 val) {
@@ -402,6 +403,7 @@ struct TPDiskMon {
case ErrorNoDeviceWithSuchSerial: return "ErrorNoDeviceWithSuchSerial";
case ErrorDeviceSerialMismatch: return "ErrorDeviceSerialMismatch";
case ErrorFake: return "ErrorFake";
+ case BootingReencryptingFormat: return "BootingReencryptingFormat";
default: return "Unknown";
}
}
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp
index 84f51fce940..34d062cce53 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.cpp
@@ -112,7 +112,7 @@ void FormatPDisk(TString path, ui64 diskSizeBytes, ui32 sectorSizeBytes, ui32 us
chunkKey, logKey, sysLogKey, mainKey, textMessage, isErasureEncodeUserLog, trimEntireDevice);
}
-bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TKey &mainKey, TPDiskInfo &outInfo,
+bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TMainKey &mainKey, TPDiskInfo &outInfo,
const bool doLock, TIntrusivePtr<NPDisk::TSectorMap> sectorMap) {
const TIntrusivePtr<::NMonitoring::TDynamicCounters> counters(new ::NMonitoring::TDynamicCounters);
auto mon = std::make_unique<TPDiskMon>(counters, 0, nullptr);
@@ -146,72 +146,74 @@ bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TKey &mainKey, TPDis
blockDevice->PreadSync(formatRaw->Data(), formatSectorsSize, 0,
NPDisk::TReqId(NPDisk::TReqId::ReadFormatInfo, 0), {});
- NPDisk::TPDiskStreamCypher cypher(true); // Format record is always encrypted
- cypher.SetKey(mainKey);
- bool isOk = false;
- alignas(16) NPDisk::TDiskFormat format;
- for (ui32 recordIdx = 0; recordIdx < NPDisk::ReplicationFactor; ++recordIdx) {
- ui64 recordSectorOffset = recordIdx * NPDisk::FormatSectorSize;
- ui8 *formatSector = formatRaw->Data() + recordSectorOffset;
- NPDisk::TDataSectorFooter *footer = (NPDisk::TDataSectorFooter*)
- (formatSector + NPDisk::FormatSectorSize - sizeof(NPDisk::TDataSectorFooter));
-
- cypher.StartMessage(footer->Nonce);
-
- alignas(16) NPDisk::TDiskFormatSector formatCandidate;
- cypher.Encrypt(formatCandidate.Raw, formatSector, NPDisk::FormatSectorSize);
-
- if (formatCandidate.Format.IsHashOk(NPDisk::FormatSectorSize)) {
- format.UpgradeFrom(formatCandidate.Format);
- isOk = true;
+ for (auto& key : mainKey) {
+ NPDisk::TPDiskStreamCypher cypher(true); // Format record is always encrypted
+ cypher.SetKey(key);
+ bool isOk = false;
+ alignas(16) NPDisk::TDiskFormat format;
+ for (ui32 recordIdx = 0; recordIdx < NPDisk::ReplicationFactor; ++recordIdx) {
+ ui64 recordSectorOffset = recordIdx * NPDisk::FormatSectorSize;
+ ui8 *formatSector = formatRaw->Data() + recordSectorOffset;
+ NPDisk::TDataSectorFooter *footer = (NPDisk::TDataSectorFooter*)
+ (formatSector + NPDisk::FormatSectorSize - sizeof(NPDisk::TDataSectorFooter));
+
+ cypher.StartMessage(footer->Nonce);
+
+ alignas(16) NPDisk::TDiskFormatSector formatCandidate;
+ cypher.Encrypt(formatCandidate.Raw, formatSector, NPDisk::FormatSectorSize);
+
+ if (formatCandidate.Format.IsHashOk(NPDisk::FormatSectorSize)) {
+ format.UpgradeFrom(formatCandidate.Format);
+ isOk = true;
+ }
}
- }
- if (isOk) {
- outInfo.Version = format.Version;
- outInfo.DiskSize = format.DiskSize;
- outInfo.SectorSizeBytes = format.SectorSize;
- outInfo.UserAccessibleChunkSizeBytes = format.GetUserAccessibleChunkSize();
- outInfo.DiskGuid = format.Guid;
- format.FormatText[sizeof(format.FormatText) - 1] = 0;
- outInfo.TextMessage = format.FormatText;
- outInfo.RawChunkSizeBytes = format.ChunkSize;
- outInfo.SysLogSectorCount = format.SysLogSectorCount;
- outInfo.SystemChunkCount = format.SystemChunkCount;
- outInfo.Timestamp = TInstant::MicroSeconds(format.TimestampUs);
- outInfo.FormatFlags = format.FormatFlagsToString(format.FormatFlags);
-
- ui32 sysLogSize = format.SectorSize * format.SysLogSectorCount * 3;
- ui32 formatBytes = NPDisk::FormatSectorSize * NPDisk::ReplicationFactor;
- ui32 sysLogOffsetSectors = (formatBytes + format.SectorSize - 1) / format.SectorSize;
- ui32 sysLogOffset = sysLogOffsetSectors * format.SectorSize;
-
- NPDisk::TAlignedData sysLogRaw(sysLogSize);
- NPDisk::TBuffer::TPtr buffer(bufferPool->Pop());
- const ui32 bufferSize = AlignDown(bufferPool->GetBufferSize(), format.SectorSize);
- const ui32 sysLogRawParts = (sysLogSize + bufferSize - 1) / bufferSize;
- for (ui32 i = 0; i < sysLogRawParts; i++) {
- const ui32 sysLogPartSize = Min(bufferSize, sysLogSize - i * bufferSize);
- Y_VERIFY(buffer->Size() >= sysLogPartSize);
- blockDevice->PreadSync(buffer->Data(), sysLogPartSize, sysLogOffset + i * bufferSize,
- NPDisk::TReqId(NPDisk::TReqId::ReadSysLogData, 0), {});
- memcpy(sysLogRaw.Get() + i * bufferSize, buffer->Data(), sysLogPartSize);
- }
+ if (isOk) {
+ outInfo.Version = format.Version;
+ outInfo.DiskSize = format.DiskSize;
+ outInfo.SectorSizeBytes = format.SectorSize;
+ outInfo.UserAccessibleChunkSizeBytes = format.GetUserAccessibleChunkSize();
+ outInfo.DiskGuid = format.Guid;
+ format.FormatText[sizeof(format.FormatText) - 1] = 0;
+ outInfo.TextMessage = format.FormatText;
+ outInfo.RawChunkSizeBytes = format.ChunkSize;
+ outInfo.SysLogSectorCount = format.SysLogSectorCount;
+ outInfo.SystemChunkCount = format.SystemChunkCount;
+ outInfo.Timestamp = TInstant::MicroSeconds(format.TimestampUs);
+ outInfo.FormatFlags = format.FormatFlagsToString(format.FormatFlags);
+
+ ui32 sysLogSize = format.SectorSize * format.SysLogSectorCount * 3;
+ ui32 formatBytes = NPDisk::FormatSectorSize * NPDisk::ReplicationFactor;
+ ui32 sysLogOffsetSectors = (formatBytes + format.SectorSize - 1) / format.SectorSize;
+ ui32 sysLogOffset = sysLogOffsetSectors * format.SectorSize;
+
+ NPDisk::TAlignedData sysLogRaw(sysLogSize);
+ NPDisk::TBuffer::TPtr buffer(bufferPool->Pop());
+ const ui32 bufferSize = AlignDown(bufferPool->GetBufferSize(), format.SectorSize);
+ const ui32 sysLogRawParts = (sysLogSize + bufferSize - 1) / bufferSize;
+ for (ui32 i = 0; i < sysLogRawParts; i++) {
+ const ui32 sysLogPartSize = Min(bufferSize, sysLogSize - i * bufferSize);
+ Y_VERIFY(buffer->Size() >= sysLogPartSize);
+ blockDevice->PreadSync(buffer->Data(), sysLogPartSize, sysLogOffset + i * bufferSize,
+ NPDisk::TReqId(NPDisk::TReqId::ReadSysLogData, 0), {});
+ memcpy(sysLogRaw.Get() + i * bufferSize, buffer->Data(), sysLogPartSize);
+ }
- outInfo.SectorInfo.clear();
- for (ui32 idx = 0; idx < format.SysLogSectorCount * 3; ++idx) {
- ui64 logSectorOffset = (ui64)idx * (ui64)format.SectorSize;
- ui8 *sector = sysLogRaw.Get() + logSectorOffset;
- NPDisk::TDataSectorFooter *logFooter = (NPDisk::TDataSectorFooter*)
- (sector + format.SectorSize - sizeof(NPDisk::TDataSectorFooter));
-
- ui64 sectorOffset = sysLogOffset + (ui64)((idx / 3) * 3) * (ui64)format.SectorSize;
- bool isCrcOk = NPDisk::TPDiskHashCalculator(KIKIMR_PDISK_ENABLE_T1HA_HASH_WRITING).CheckSectorHash(
- sectorOffset, format.MagicSysLogChunk, sector, format.SectorSize, logFooter->Hash);
- outInfo.SectorInfo.push_back(TPDiskInfo::TSectorInfo(logFooter->Nonce, logFooter->Version, isCrcOk));
- }
+ outInfo.SectorInfo.clear();
+ for (ui32 idx = 0; idx < format.SysLogSectorCount * 3; ++idx) {
+ ui64 logSectorOffset = (ui64)idx * (ui64)format.SectorSize;
+ ui8 *sector = sysLogRaw.Get() + logSectorOffset;
+ NPDisk::TDataSectorFooter *logFooter = (NPDisk::TDataSectorFooter*)
+ (sector + format.SectorSize - sizeof(NPDisk::TDataSectorFooter));
+
+ ui64 sectorOffset = sysLogOffset + (ui64)((idx / 3) * 3) * (ui64)format.SectorSize;
+ bool isCrcOk = NPDisk::TPDiskHashCalculator(KIKIMR_PDISK_ENABLE_T1HA_HASH_WRITING).CheckSectorHash(
+ sectorOffset, format.MagicSysLogChunk, sector, format.SectorSize, logFooter->Hash);
+ outInfo.SectorInfo.push_back(TPDiskInfo::TSectorInfo(logFooter->Nonce, logFooter->Version, isCrcOk));
+ }
- return true;
+ return true;
+ }
}
TStringStream str;
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h
index 89b8f6418e9..47021792407 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_tools.h
@@ -46,7 +46,7 @@ void FormatPDisk(TString path, ui64 diskSizeBytes, ui32 sectorSizeBytes, ui32 us
const bool isErasureEncodeUserLog = false, const bool trimEntireDevice = false,
TIntrusivePtr<NPDisk::TSectorMap> sectorMap = nullptr);
-bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TKey &mainKey, TPDiskInfo &outInfo,
+bool ReadPDiskFormatInfo(const TString &path, const NPDisk::TMainKey &mainKey, TPDiskInfo &outInfo,
const bool doLock = false, TIntrusivePtr<NPDisk::TSectorMap> sectorMap = nullptr);
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp
index 4266d75266d..e89c9151b4b 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp
@@ -674,7 +674,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
NKikimrProto::OK);
round = evInitRes->PDiskParams->OwnerRound + 1;
- testCtx.MainKey += 123;
+ testCtx.MainKey[0] += 123;
testCtx.UpdateConfigRecreatePDisk(testCtx.GetPDiskConfig());
evInitRes = testCtx.TestResponce<NPDisk::TEvYardInitResult>(
@@ -684,5 +684,93 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
new NPDisk::TEvCheckSpace(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound),
NKikimrProto::CORRUPTED);
}
+
+ Y_UNIT_TEST(ChangePDiskKey) {
+ const TString data = PrepareData(4096);
+
+ TActorTestContext testCtx(false);
+
+ TVDiskMock mock(&testCtx);
+ mock.InitFull();
+
+ mock.ReserveChunk();
+ const ui32 chunk = *mock.Chunks[EChunkState::RESERVED].begin();
+
+ auto readChunk = [&]() {
+ auto evReadRes = testCtx.TestResponce<NPDisk::TEvChunkReadResult>(
+ new NPDisk::TEvChunkRead(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound,
+ chunk, 0, data.size(), 0, nullptr),
+ NKikimrProto::OK);
+ UNIT_ASSERT_VALUES_EQUAL(evReadRes->Data.ToString(), data);
+ };
+
+ TString dataCopy = data;
+ testCtx.TestResponce<NPDisk::TEvChunkWriteResult>(new NPDisk::TEvChunkWrite(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound,
+ chunk, 0, new NPDisk::TEvChunkWrite::TStrokaBackedUpParts(dataCopy), nullptr, false, 0),
+ NKikimrProto::OK);
+ mock.CommitReservedChunks();
+
+ readChunk();
+
+ testCtx.MainKey.push_back(0xFull);
+ testCtx.RestartPDiskSync();
+ mock.InitFull();
+ readChunk();
+
+ testCtx.MainKey = { 0xFull };
+ testCtx.RestartPDiskSync();
+ mock.InitFull();
+ readChunk();
+
+ testCtx.MainKey = { 0xFull, 0xA };
+ testCtx.RestartPDiskSync();
+ mock.InitFull();
+ readChunk();
+
+ testCtx.MainKey = { 0xFull, 0xA, 0xB, 0xC };
+ testCtx.RestartPDiskSync();
+ mock.InitFull();
+ readChunk();
+
+ testCtx.MainKey = { 0xC };
+ testCtx.RestartPDiskSync();
+ mock.InitFull();
+ readChunk();
+ }
+
+
+ Y_UNIT_TEST(WrongPDiskKey) {
+ const TString data = PrepareData(4096);
+
+ TActorTestContext testCtx(false);
+
+ TVDiskMock mock(&testCtx);
+ mock.InitFull();
+
+ mock.ReserveChunk();
+ const ui32 chunk = *mock.Chunks[EChunkState::RESERVED].begin();
+
+ TString dataCopy = data;
+ testCtx.TestResponce<NPDisk::TEvChunkWriteResult>(new NPDisk::TEvChunkWrite(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound,
+ chunk, 0, new NPDisk::TEvChunkWrite::TStrokaBackedUpParts(dataCopy), nullptr, false, 0),
+ NKikimrProto::OK);
+ mock.CommitReservedChunks();
+ testCtx.TestResponce<NPDisk::TEvCheckSpaceResult>(
+ new NPDisk::TEvCheckSpace(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound),
+ NKikimrProto::OK);
+ testCtx.TestResponce<NPDisk::TEvChunkReadResult>(
+ new NPDisk::TEvChunkRead(mock.PDiskParams->Owner, mock.PDiskParams->OwnerRound,
+ chunk, 0, data.size(), 0, nullptr),
+ NKikimrProto::OK);
+
+ testCtx.MainKey = { 0xABCDEF };
+ testCtx.TestResponce<NPDisk::TEvYardControlResult>(
+ new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStop, nullptr),
+ NKikimrProto::OK);
+
+ testCtx.TestResponce<NPDisk::TEvYardControlResult>(
+ new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStart, (void*)(&testCtx.MainKey)),
+ NKikimrProto::CORRUPTED);
+ }
}
} // namespace NKikimr
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h
index 6cf9ceb49bf..f6cf4482d01 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h
@@ -23,7 +23,7 @@ private:
public:
TActorId Sender;
- NPDisk::TKey MainKey = NPDisk::YdbDefaultPDiskSequence;
+ NPDisk::TMainKey MainKey = { NPDisk::YdbDefaultPDiskSequence };
TTestContext TestCtx{false, /*use sector map*/ true};
TIntrusivePtr<TPDiskConfig> DefaultPDiskConfig(bool isBad) {
@@ -97,7 +97,7 @@ public:
if (!PDisk && !UsePDiskMock) {
// To be sure that pdisk actor is in StateOnline
TestResponce<NPDisk::TEvYardControlResult>(
- new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStart, &MainKey),
+ new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStart, (void*)(&MainKey)),
NKikimrProto::OK);
const auto evControlRes = TestResponce<NPDisk::TEvYardControlResult>(
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp
index a63d37d5c50..ee72a048938 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.cpp
@@ -87,8 +87,9 @@ void Run(TVector<IActor*> tests, TTestRunConfig runCfg) {
pDiskConfig->EnableSectorEncryption = !pDiskConfig->SectorMap;
pDiskConfig->UseT1ha0HashInFooter = runCfg.UseT1ha0Hasher;
+ NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence};
TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(),
- NPDisk::YdbDefaultPDiskSequence, mainCounters), TMailboxType::Revolving, 0);
+ mainKey, mainCounters), TMailboxType::Revolving, 0);
setup1->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(pDiskId, pDiskSetup));
for (ui32 i = 0; i < runCfg.Instances; ++i) {
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp
index ba7353ad62e..1f98f7a952f 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_yard.cpp
@@ -534,7 +534,8 @@ YARD_UNIT_TEST(TestDamagedFirstRecordToKeep) {
MakeDirIfNotExist(databaseDirectory.c_str());
}
TPDiskInfo info;
- bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap);
+ const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence};
+ bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap);
UNIT_ASSERT_VALUES_EQUAL(isOk, true);
ui32 dataSize = info.SystemChunkCount * info.RawChunkSizeBytes;
@@ -880,7 +881,8 @@ YARD_UNIT_TEST(TestFormatInfo) {
FormatPDiskForTest(dataPath, tc.PDiskGuid, chunkSize, 1 << 30, false, tc.SectorMap);
TPDiskInfo info;
- bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap);
+ const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence};
+ bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap);
UNIT_ASSERT_VALUES_EQUAL(isOk, true);
UNIT_ASSERT_VALUES_EQUAL(info.TextMessage, "Info");
}
@@ -914,7 +916,8 @@ YARD_UNIT_TEST(TestRestartAtNonceJump) {
MakeDirIfNotExist(databaseDirectory.c_str());
}
TPDiskInfo info;
- bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap);
+ const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence};
+ bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap);
UNIT_ASSERT_VALUES_EQUAL(isOk, true);
// Destroy data in chunks starting at# SystemChunkCount + 1
ui32 dataSize = 8 * chunkSize;
@@ -943,7 +946,8 @@ YARD_UNIT_TEST(TestRestartAtChunkEnd) {
MakeDirIfNotExist(databaseDirectory.c_str());
}
TPDiskInfo info;
- bool isOk = ReadPDiskFormatInfo(dataPath, NPDisk::YdbDefaultPDiskSequence, info, false, tc.SectorMap);
+ const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence};
+ bool isOk = ReadPDiskFormatInfo(dataPath, mainKey, info, false, tc.SectorMap);
UNIT_ASSERT_VALUES_EQUAL(isOk, true);
// Destroy data in chunks starting at# SystemChunkCount + 1
ui32 dataSize = 8 * chunkSize;
diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
index 0f42392943f..1f7fa20c6e3 100644
--- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h
+++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
@@ -46,7 +46,7 @@ struct TEnvironmentSetup {
{}
void Create(const TActorContext& ctx, ui32 pdiskId, const TIntrusivePtr<TPDiskConfig>& cfg,
- const NPDisk::TKey& /*mainKey*/, ui32 poolId, ui32 nodeId) override {
+ const NPDisk::TMainKey& /*mainKey*/, ui32 poolId, ui32 nodeId) override {
const auto key = std::make_pair(nodeId, pdiskId);
TIntrusivePtr<TPDiskMockState>& state = Env.PDiskMockStates[key];
if (!state) {
diff --git a/ydb/core/blobstorage/ut_group/main.cpp b/ydb/core/blobstorage/ut_group/main.cpp
index 0ef59b5479c..d703975dcf1 100644
--- a/ydb/core/blobstorage/ut_group/main.cpp
+++ b/ydb/core/blobstorage/ut_group/main.cpp
@@ -161,7 +161,7 @@ public:
};
TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters;
- const NPDisk::TKey MainKey = NPDisk::YdbDefaultPDiskSequence;
+ const NPDisk::TMainKey MainKey = { NPDisk::YdbDefaultPDiskSequence };
const ui32 NodeCount;
const ui32 GroupId = 0;
std::vector<TDiskRecord> Disks;
diff --git a/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h b/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h
index 06b5d5a0ba8..f9e1bcf05bb 100644
--- a/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h
+++ b/ydb/core/blobstorage/ut_pdiskfit/lib/basic_test.h
@@ -85,7 +85,7 @@ public:
Y_VERIFY(ctx.ExecutorThread.ActorSystem);
Y_VERIFY(PDiskConfig);
Y_VERIFY(AppData(ctx));
- std::unique_ptr<IActor> pdiskActor(CreatePDisk(PDiskConfig, 1, Counters->GetSubgroup("subsystem", "pdisk")));
+ std::unique_ptr<IActor> pdiskActor(CreatePDisk(PDiskConfig, {1}, Counters->GetSubgroup("subsystem", "pdisk")));
const TActorId actorId = ctx.ExecutorThread.ActorSystem->Register(pdiskActor.release(), TMailboxType::Simple,
AppData(ctx)->SystemPoolId);
PDiskServiceId = MakeBlobStoragePDiskID(ctx.ExecutorThread.ActorSystem->NodeId, PDiskConfig->PDiskId);
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp b/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp
index ccee189f60b..7d691ba7a1e 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp
@@ -158,8 +158,9 @@ void TAllPDisks::ActorSetupCmd(NActors::TActorSystemSetup *setup, ui32 node,
TPDiskCategory(deviceType, 0).GetRaw()));
pDiskConfig->GetDriveDataSwitch = NKikimrBlobStorage::TPDiskConfig::DoNotTouch;
pDiskConfig->WriteCacheSwitch = NKikimrBlobStorage::TPDiskConfig::DoNotTouch;
+ const NPDisk::TMainKey mainKey = {NPDisk::YdbDefaultPDiskSequence};
TActorSetupCmd pDiskSetup(CreatePDisk(pDiskConfig.Get(),
- NPDisk::YdbDefaultPDiskSequence, counters), TMailboxType::Revolving, 0);
+ mainKey, counters), TMailboxType::Revolving, 0);
setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(inst.PDiskActorID, pDiskSetup));
}
}
diff --git a/ydb/core/driver_lib/base_utils/base_utils.h b/ydb/core/driver_lib/base_utils/base_utils.h
index 4809f2824da..3dfd02853eb 100644
--- a/ydb/core/driver_lib/base_utils/base_utils.h
+++ b/ydb/core/driver_lib/base_utils/base_utils.h
@@ -14,6 +14,8 @@
#include <library/cpp/string_utils/parse_size/parse_size.h>
#include <library/cpp/svnversion/svnversion.h>
+#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h>
+
namespace NKikimr {
struct TCmdFormatConfig {
@@ -22,7 +24,7 @@ struct TCmdFormatConfig {
NSize::TSize ChunkSize;
NSize::TSize SectorSize;
ui64 Guid;
- ui64 MainKey;
+ NPDisk::TMainKey MainKey;
TString TextMessage;
TCmdFormatConfig();
@@ -32,7 +34,8 @@ struct TCmdFormatConfig {
struct TCmdFormatInfoConfig {
TString Path;
- ui64 MainKey;
+ TVector<NPDisk::TKey> MainKeyTmp; // required for .AppendTo()
+ NPDisk::TMainKey MainKey;
bool IsVerbose;
TCmdFormatInfoConfig();
diff --git a/ydb/core/driver_lib/base_utils/format_info.cpp b/ydb/core/driver_lib/base_utils/format_info.cpp
index e5d53a7b803..03498da3756 100644
--- a/ydb/core/driver_lib/base_utils/format_info.cpp
+++ b/ydb/core/driver_lib/base_utils/format_info.cpp
@@ -58,7 +58,7 @@ int MainFormatInfo(const TCommandConfig &cmdConf, int argc, char** argv) {
}
TCmdFormatInfoConfig::TCmdFormatInfoConfig()
- : MainKey(0)
+ : MainKey({})
, IsVerbose(false)
{}
@@ -69,9 +69,9 @@ void TCmdFormatInfoConfig::Parse(int argc, char **argv) {
opts.AddLongOption('p', "pdisk-path", "path to pdisk to read format info").RequiredArgument("PATH").Required()
.StoreResult(&Path);
opts.AddLongOption('k', "main-key", "encryption main-key to use while reading").RequiredArgument("NUM")
- .Optional().StoreResult(&MainKey); // TODO: make required
+ .Optional().AppendTo(&MainKeyTmp); // TODO: make required
opts.AddLongOption("master-key", "obsolete: use main-key").RequiredArgument("NUM")
- .Optional().StoreResult(&MainKey); // TODO: remove after migration
+ .Optional().AppendTo(&MainKeyTmp); // TODO: remove after migration
opts.AddLongOption('v', "verbose", "output detailed information for debugging").Optional().NoArgument()
.SetFlag(&IsVerbose);
@@ -85,6 +85,11 @@ void TCmdFormatInfoConfig::Parse(int argc, char **argv) {
bool hasKOption = res.FindCharOptParseResult('k');
if (!hasMainOption && !hasMasterOption && !hasKOption)
ythrow yexception() << "missing main-key param";
+
+ MainKey = {};
+ for (auto& key : MainKeyTmp) {
+ MainKey.push_back(key);
+ }
}
}
diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp
index a9be370d37f..ebc4a18930c 100644
--- a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp
@@ -14,20 +14,21 @@ public:
bool IsVerbose;
bool LockDevice;
- ui64 MainKey;
+ TVector<NPDisk::TKey> MainKeyTmp; // required for .AppendTo()
+ NPDisk::TMainKey MainKey;
TString Path;
virtual void Config(TConfig& config) override {
TClientCommand::Config(config);
IsVerbose = false;
LockDevice = false;
- MainKey = 0;
+ MainKeyTmp = {};
config.SetFreeArgsNum(1);
SetFreeArgTitle(0, "<PATH>", "Disk path");
config.Opts->AddLongOption('k', "main-key", "encryption main-key to use while reading").RequiredArgument("NUM")
- .Optional().StoreResult(&MainKey); // TODO: make required
+ .Optional().AppendTo(&MainKeyTmp); // TODO: make required
config.Opts->AddLongOption("master-key", "obsolete: use main-key").RequiredArgument("NUM")
- .Optional().StoreResult(&MainKey); // TODO: remove after migration
+ .Optional().AppendTo(&MainKeyTmp); // TODO: remove after migration
config.Opts->AddLongOption('v', "verbose", "output detailed information for debugging").Optional().NoArgument()
.SetFlag(&IsVerbose);
config.Opts->AddLongOption('l', "lock", "lock device before reading disk info").Optional().NoArgument()
@@ -43,6 +44,11 @@ public:
bool hasKOption = config.ParseResult->FindCharOptParseResult('k');
if (!hasMainOption && !hasMasterOption && !hasKOption)
ythrow yexception() << "missing main-key param";
+
+ MainKey = {};
+ for (auto& key : MainKeyTmp) {
+ MainKey.push_back(key);
+ }
}
virtual int Run(TConfig&) override {
@@ -108,13 +114,13 @@ public:
NSize::TSize ChunkSize;
NSize::TSize SectorSize;
ui64 Guid;
- ui64 MainKey;
+ TVector<NPDisk::TKey> MainKeyTmp;
TString TextMessage;
bool IsErasureEncode;
virtual void Config(TConfig& config) override {
TClientCommand::Config(config);
- MainKey = 0;
+ MainKey = {};
DiskSize = 0;
ChunkSize = 128 << 20;
SectorSize = 4 << 10;
@@ -136,9 +142,9 @@ public:
.StoreResult(&Guid);
config.Opts->AddLongOption('k', "main-key", "encryption main-key to set while formatting.\n"
"Make sure you use the same master key when you format your pdisks and when you run kikimr.")
- .RequiredArgument("NUM").Optional().StoreResult(&MainKey);
+ .RequiredArgument("NUM").Optional().AppendTo(&MainKeyTmp);
config.Opts->AddLongOption("master-key", "obsolete: user main-key")
- .RequiredArgument("NUM").Optional().StoreResult(&MainKey);
+ .RequiredArgument("NUM").Optional().AppendTo(&MainKeyTmp);
config.Opts->AddLongOption('t', "text-message", "text message to store in format sector (up to 4000 characters long)")
.OptionalArgument("STR").Optional().StoreResult(&TextMessage);
config.Opts->AddLongOption('e', "erasure-encode", "erasure-encode data to recover from single-sector failures")
@@ -153,6 +159,7 @@ public:
NPDisk::TKey ChunkKey;
NPDisk::TKey LogKey;
NPDisk::TKey SysLogKey;
+ NPDisk::TMainKey MainKey;
virtual void Parse(TConfig& config) override {
TClientCommand::Parse(config);
@@ -165,10 +172,14 @@ public:
bool hasKOption = config.ParseResult->FindCharOptParseResult('k');
if (!hasMainOption && !hasMasterOption && !hasKOption)
ythrow yexception() << "missing main-key param";
+
+ for (auto& key : MainKeyTmp) {
+ MainKey.push_back(key);
+ }
}
virtual int Run(TConfig&) override {
- FormatPDisk(Path, DiskSize, SectorSize, ChunkSize, Guid, ChunkKey, LogKey, SysLogKey, MainKey, TextMessage,
+ FormatPDisk(Path, DiskSize, SectorSize, ChunkSize, Guid, ChunkKey, LogKey, SysLogKey, MainKey.back(), TextMessage,
IsErasureEncode);
return 0;
}
diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp
index fc3ab1e96eb..865f975bef0 100644
--- a/ydb/core/driver_lib/run/config_parser.cpp
+++ b/ydb/core/driver_lib/run/config_parser.cpp
@@ -76,6 +76,7 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts&
opts.AddLongOption("alloc-file", "Allocator config file").OptionalArgument("PATH");
opts.AddLongOption("yql-file", "Yql Analytics config file").OptionalArgument("PATH");
opts.AddLongOption("yq-file", "Yandex Query config file").OptionalArgument("PATH");
+ opts.AddLongOption("pdisk-key-file", "pdisk encryption key config file").OptionalArgument("PATH");
}
void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResult& res) {
diff --git a/ydb/core/testlib/basics/helpers.cpp b/ydb/core/testlib/basics/helpers.cpp
index 259e4f9da94..ccc020812ce 100644
--- a/ydb/core/testlib/basics/helpers.cpp
+++ b/ydb/core/testlib/basics/helpers.cpp
@@ -67,7 +67,7 @@ namespace NKikimr {
};
void TStrandedPDiskServiceFactory::Create(const TActorContext &ctx, ui32 pDiskID,
- const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId)
+ const TIntrusivePtr<TPDiskConfig> &cfg, const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId)
{
Y_UNUSED(ctx);
Y_VERIFY(!Runtime.IsRealThreads());
diff --git a/ydb/core/testlib/basics/helpers.h b/ydb/core/testlib/basics/helpers.h
index c71e7d84f69..9658b837149 100644
--- a/ydb/core/testlib/basics/helpers.h
+++ b/ydb/core/testlib/basics/helpers.h
@@ -68,7 +68,7 @@ namespace NFake {
{}
void Create(const TActorContext &ctx, ui32 pDiskID, const TIntrusivePtr<TPDiskConfig> &cfg,
- const NPDisk::TKey &mainKey, ui32 poolId, ui32 nodeId) override;
+ const NPDisk::TMainKey &mainKey, ui32 poolId, ui32 nodeId) override;
virtual ~TStrandedPDiskServiceFactory()
{}