Add read and write device performance estimations for different device types, KIKIMR-18896

Add distinct read and write SectorMap performance limitations

5 files changed, 127 insertions, 35 deletions

diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
index e2f726d2660..9463b727c26 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
@@ -65,8 +65,10 @@ TPDisk::TPDisk(const TIntrusivePtr<TPDiskConfig> cfg, const TIntrusivePtr<::NMon
     if (Cfg->SectorMap) {
         auto diskModeParams = Cfg->SectorMap->GetDiskModeParams();
         if (diskModeParams) {
-            SectorMapFirstSectorRate = TControlWrapper(diskModeParams->FirstSectorRate, 0, 100000ull * 1024 * 1024);
-            SectorMapLastSectorRate = TControlWrapper(diskModeParams->LastSectorRate, 0, 100000ull * 1024 * 1024);
+            SectorMapFirstSectorReadRate = TControlWrapper(diskModeParams->FirstSectorReadRate, 0, 100000ull * 1024 * 1024);
+            SectorMapLastSectorReadRate = TControlWrapper(diskModeParams->LastSectorReadRate, 0, 100000ull * 1024 * 1024);
+            SectorMapFirstSectorWriteRate = TControlWrapper(diskModeParams->FirstSectorWriteRate, 0, 100000ull * 1024 * 1024);
+            SectorMapLastSectorWriteRate = TControlWrapper(diskModeParams->LastSectorWriteRate, 0, 100000ull * 1024 * 1024);
             SectorMapSeekSleepMicroSeconds = TControlWrapper(diskModeParams->SeekSleepMicroSeconds, 0, 100ul * 1000 * 1000);
         }
     }
@@ -2557,11 +2559,14 @@ bool TPDisk::Initialize(TActorSystem *actorSystem, const TActorId &pDiskActor) {
             if (Cfg->SectorMap) {
                 auto diskModeParams = Cfg->SectorMap->GetDiskModeParams();
                 if (diskModeParams) {
-                    REGISTER_LOCAL_CONTROL(SectorMapFirstSectorRate);
-                    REGISTER_LOCAL_CONTROL(SectorMapLastSectorRate);
+                    REGISTER_LOCAL_CONTROL(SectorMapFirstSectorReadRate);
+                    REGISTER_LOCAL_CONTROL(SectorMapLastSectorReadRate);
+                    REGISTER_LOCAL_CONTROL(SectorMapFirstSectorWriteRate);
+                    REGISTER_LOCAL_CONTROL(SectorMapLastSectorWriteRate);
                     REGISTER_LOCAL_CONTROL(SectorMapSeekSleepMicroSeconds);
 
-                    LastSectorRateControlName = TStringBuilder() << "PDisk_" << PDiskId << "_SectorMapLastSectorRate";
+                    LastSectorReadRateControlName = TStringBuilder() << "PDisk_" << PDiskId << "_SectorMapLastSectorReadRate";
+                    LastSectorWriteRateControlName = TStringBuilder() << "PDisk_" << PDiskId << "_SectorMapLastSectorWriteRate";
                 }
             }
         }
@@ -3557,13 +3562,22 @@ void TPDisk::Update() {
     if (Cfg->SectorMap) {
         auto diskModeParams = Cfg->SectorMap->GetDiskModeParams();
         if (diskModeParams) {
-            diskModeParams->FirstSectorRate.store(SectorMapFirstSectorRate);
-            if (SectorMapFirstSectorRate < SectorMapLastSectorRate) {
-                TAtomic prevValue;
-                ActorSystem->AppData<TAppData>()->Icb->SetValue(LastSectorRateControlName, SectorMapFirstSectorRate, prevValue);
-                diskModeParams->LastSectorRate.store(SectorMapFirstSectorRate);
+            TAtomic prevValue;
+
+            diskModeParams->FirstSectorReadRate.store(SectorMapFirstSectorReadRate);
+            if (SectorMapFirstSectorReadRate < SectorMapLastSectorReadRate) {
+                ActorSystem->AppData<TAppData>()->Icb->SetValue(LastSectorReadRateControlName, SectorMapFirstSectorReadRate, prevValue);
+                diskModeParams->LastSectorReadRate.store(SectorMapFirstSectorReadRate);
+            } else {
+                diskModeParams->LastSectorReadRate.store(SectorMapLastSectorReadRate);
+            }
+
+            diskModeParams->FirstSectorWriteRate.store(SectorMapFirstSectorWriteRate);
+            if (SectorMapFirstSectorWriteRate < SectorMapLastSectorWriteRate) {
+                ActorSystem->AppData<TAppData>()->Icb->SetValue(LastSectorWriteRateControlName, SectorMapFirstSectorWriteRate, prevValue);
+                diskModeParams->LastSectorWriteRate.store(SectorMapFirstSectorWriteRate);
             } else {
-                diskModeParams->LastSectorRate.store(SectorMapLastSectorRate);
+                diskModeParams->LastSectorWriteRate.store(SectorMapLastSectorWriteRate);
             }
 
             diskModeParams->SeekSleepMicroSeconds.store(SectorMapSeekSleepMicroSeconds);
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
index 800f8c2a184..a3feee5bfd3 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h
@@ -99,11 +99,14 @@ public:
     TControlWrapper ForsetiOpPieceSizeRot;
 
     // SectorMap Controls
-    TControlWrapper SectorMapFirstSectorRate;
-    TControlWrapper SectorMapLastSectorRate;
-    // to update if SectorMapFirstSectorRate < SectorMapLastSectorRate
-    TString LastSectorRateControlName;
+    TControlWrapper SectorMapFirstSectorReadRate;
+    TControlWrapper SectorMapLastSectorReadRate;
+    TControlWrapper SectorMapFirstSectorWriteRate;
+    TControlWrapper SectorMapLastSectorWriteRate;
     TControlWrapper SectorMapSeekSleepMicroSeconds;
+    // used to store valid value in ICB if SectorMapFirstSector*Rate < SectorMapLastSector*Rate
+    TString LastSectorReadRateControlName;
+    TString LastSectorWriteRateControlName;
 
     ui64 ForsetiMinLogCostNs = 2000000ull;
     i64 ForsetiMaxLogBatchNsCached;
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_sectormap.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_sectormap.cpp
index 232869c3824..c32f4cb9f89 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_sectormap.cpp
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_sectormap.cpp
@@ -19,9 +19,16 @@ Y_UNIT_TEST_SUITE(TSectorMap) {
         ui64 dataSize = dataSizeMb * 1024 * 1024;
         ui64 deviceSize = diskSizeGb * 1024 * 1024 * 1024;
 
-        ui64 diskRate = toFirstSector ? 
-            NPDisk::NSectorMap::DiskModeParamPresets[(ui32)diskMode][1] : 
-            NPDisk::NSectorMap::DiskModeParamPresets[(ui32)diskMode][2];
+        auto deviceType = NPDisk::NSectorMap::DiskModeToDeviceType(diskMode);
+        ui64 diskRate;
+        if (testRead) {
+            diskRate = toFirstSector ? NPDisk::DevicePerformance.at(deviceType).FirstSectorReadBytesPerSec : 
+                NPDisk::DevicePerformance.at(deviceType).LastSectorReadBytesPerSec;
+        } else {
+            diskRate = toFirstSector ? NPDisk::DevicePerformance.at(deviceType).FirstSectorWriteBytesPerSec : 
+                NPDisk::DevicePerformance.at(deviceType).LastSectorWriteBytesPerSec;
+        }
+
         ui64 sectorsNum = deviceSize / NPDisk::NSectorMap::SECTOR_SIZE;
         ui64 sectorPos = toFirstSector ? 0 : sectorsNum - dataSize / NPDisk::NSectorMap::SECTOR_SIZE - 2;
         double timeExpected = (double)dataSize / diskRate;
diff --git a/ydb/library/pdisk_io/device_type.h b/ydb/library/pdisk_io/device_type.h
index 56721512174..c4e8bdc59b7 100644
--- a/ydb/library/pdisk_io/device_type.h
+++ b/ydb/library/pdisk_io/device_type.h
@@ -3,6 +3,8 @@
 #include <util/generic/string.h>
 #include <util/system/types.h>
 
+#include <unordered_map>
+
 namespace NKikimr::NPDisk {
     enum EDeviceType : ui8 {
         DEVICE_TYPE_ROT = 0,
@@ -11,6 +13,45 @@ namespace NKikimr::NPDisk {
         DEVICE_TYPE_UNKNOWN = 255,
     };
 
+    struct TDevicePerformanceParams {
+        ui64 SeekTimeNs;
+        ui64 FirstSectorReadBytesPerSec;
+        ui64 LastSectorReadBytesPerSec;
+        ui64 FirstSectorWriteBytesPerSec;
+        ui64 LastSectorWriteBytesPerSec;
+    };
+
+    const static std::unordered_map<EDeviceType, TDevicePerformanceParams> DevicePerformance = {
+        { DEVICE_TYPE_UNKNOWN, TDevicePerformanceParams{
+            .SeekTimeNs = 0,
+            .FirstSectorReadBytesPerSec = 0,
+            .LastSectorReadBytesPerSec = 0,
+            .FirstSectorWriteBytesPerSec = 0,
+            .LastSectorWriteBytesPerSec = 0,
+        } },
+        { DEVICE_TYPE_ROT, TDevicePerformanceParams{
+            .SeekTimeNs = 9000,
+            .FirstSectorReadBytesPerSec = 200ull * 1024 * 1024,
+            .LastSectorReadBytesPerSec = 66ull * 1024 * 1024,
+            .FirstSectorWriteBytesPerSec = 200ull * 1024 * 1024,
+            .LastSectorWriteBytesPerSec = 66ull * 1024 * 1024,
+        } },
+        { DEVICE_TYPE_SSD, TDevicePerformanceParams{
+            .SeekTimeNs = 0,
+            .FirstSectorReadBytesPerSec = 500ull * 1024 * 1024,
+            .LastSectorReadBytesPerSec = 500ull * 1024 * 1024,
+            .FirstSectorWriteBytesPerSec = 500ull * 1024 * 1024,
+            .LastSectorWriteBytesPerSec = 500ull * 1024 * 1024,
+        } },
+        { DEVICE_TYPE_NVME, TDevicePerformanceParams{
+            .SeekTimeNs = 0,
+            .FirstSectorReadBytesPerSec = 1000ull * 1024 * 1024,
+            .LastSectorReadBytesPerSec = 1000ull * 1024 * 1024,
+            .FirstSectorWriteBytesPerSec = 1000ull * 1024 * 1024,
+            .LastSectorWriteBytesPerSec = 1000ull * 1024 * 1024,
+        } },
+    };
+
     TString DeviceTypeStr(const EDeviceType type, bool isShort);
     EDeviceType DeviceTypeFromStr(const TString &typeName);
 
diff --git a/ydb/library/pdisk_io/sector_map.h b/ydb/library/pdisk_io/sector_map.h
index f1143211a3c..0fb2f97d9af 100644
--- a/ydb/library/pdisk_io/sector_map.h
+++ b/ydb/library/pdisk_io/sector_map.h
@@ -20,6 +20,8 @@
 #include <atomic>
 #include <optional>
 
+#include "device_type.h"
+
 namespace NKikimr {
 namespace NPDisk {
 
@@ -62,14 +64,31 @@ inline TString DiskModeToString(EDiskMode diskMode) {
     }
 }
 
-static constexpr std::array<std::array<ui64, 3>, NSectorMap::DM_COUNT> DiskModeParamPresets = {
-    {
-        {0, 0, 0}, // DM_NONE
-        {9000, 200ull * 1024 * 1024, 66ull * 1024 * 1024}, // DM_HDD
-        {0, 500ull * 1024 * 1024, 500ull * 1024 * 1024}, // DM_SSD
-        {0, 1000ull * 1024 * 1024, 1000ull * 1024 * 1024}, // DM_NVME, probably unusable
+inline EDiskMode DiskModeFromDeviceType(const EDeviceType& deviceType) {
+    switch (deviceType) {
+    case DEVICE_TYPE_ROT:
+        return DM_HDD;
+    case DEVICE_TYPE_SSD:
+        return DM_SSD;
+    case DEVICE_TYPE_NVME:
+        return DM_NVME;
+    default:
+        return DM_NONE;
     }
-};
+}
+
+inline EDeviceType DiskModeToDeviceType(const EDiskMode& diskMode) {
+    switch (diskMode) {
+    case DM_HDD:
+        return DEVICE_TYPE_ROT;
+    case DM_SSD:
+        return DEVICE_TYPE_SSD;
+    case DM_NVME:
+        return DEVICE_TYPE_NVME;
+    default:
+        return DEVICE_TYPE_UNKNOWN;
+    }
+}
 
 constexpr ui64 SECTOR_SIZE = 4096;
 
@@ -79,8 +98,10 @@ class TSectorOperationThrottler {
 public:
     struct TDiskModeParams {
         std::atomic<ui64> SeekSleepMicroSeconds;
-        std::atomic<ui64> FirstSectorRate;
-        std::atomic<ui64> LastSectorRate;
+        std::atomic<ui64> FirstSectorReadRate;
+        std::atomic<ui64> LastSectorReadRate;
+        std::atomic<ui64> FirstSectorWriteRate;
+        std::atomic<ui64> LastSectorWriteRate;
     };
 
 public:
@@ -91,21 +112,26 @@ public:
     void Init(ui64 sectors, NSectorMap::EDiskMode diskMode) {
         Y_VERIFY(sectors > 0);
 
-        Y_VERIFY((ui32)diskMode < DiskModeParamPresets.size());
-        DiskModeParams.SeekSleepMicroSeconds = DiskModeParamPresets[diskMode][0];
-        DiskModeParams.FirstSectorRate = DiskModeParamPresets[diskMode][1];
-        DiskModeParams.LastSectorRate = DiskModeParamPresets[diskMode][2];
+        Y_VERIFY((ui32)diskMode < DM_COUNT);
+        EDeviceType deviceType = DiskModeToDeviceType(diskMode);
+        DiskModeParams.SeekSleepMicroSeconds = DevicePerformance.at(deviceType).SeekTimeNs;
+        DiskModeParams.FirstSectorReadRate = DevicePerformance.at(deviceType).FirstSectorReadBytesPerSec;
+        DiskModeParams.LastSectorReadRate = DevicePerformance.at(deviceType).LastSectorReadBytesPerSec;
+        DiskModeParams.FirstSectorWriteRate = DevicePerformance.at(deviceType).FirstSectorWriteBytesPerSec;
+        DiskModeParams.LastSectorWriteRate = DevicePerformance.at(deviceType).LastSectorWriteBytesPerSec;
 
         MaxSector = sectors - 1;
         MostRecentlyUsedSector = 0;
     }
 
     void ThrottleRead(i64 size, ui64 offset, bool prevOperationIsInProgress, double operationTimeMs) {
-        ThrottleOperation(size, offset, prevOperationIsInProgress, operationTimeMs);
+        ThrottleOperation(size, offset, prevOperationIsInProgress, operationTimeMs,
+                DiskModeParams.FirstSectorReadRate, DiskModeParams.LastSectorReadRate);
     }
 
     void ThrottleWrite(i64 size, ui64 offset, bool prevOperationIsInProgress, double operationTimeMs) {
-        ThrottleOperation(size, offset, prevOperationIsInProgress, operationTimeMs);
+        ThrottleOperation(size, offset, prevOperationIsInProgress, operationTimeMs,
+                DiskModeParams.FirstSectorWriteRate, DiskModeParams.LastSectorWriteRate);
     }
 
     TDiskModeParams* GetDiskModeParams() {
@@ -114,7 +140,8 @@ public:
 
 private:
     /* throttle read/write operation */
-    void ThrottleOperation(i64 size, ui64 offset, bool prevOperationIsInProgress, double operationTimeMs) {
+    void ThrottleOperation(i64 size, ui64 offset, bool prevOperationIsInProgress, double operationTimeMs,
+            ui64 firstSectorRate, ui64 lastSectorRate) {
         if (size == 0) {
             return;
         }
@@ -132,7 +159,7 @@ private:
             MostRecentlyUsedSector = endSector - 1;
         }
 
-        auto rate = CalcRate(DiskModeParams.FirstSectorRate, DiskModeParams.LastSectorRate, midSector, MaxSector);
+        auto rate = CalcRate(firstSectorRate, lastSectorRate, midSector, MaxSector);
 
         auto rateByMilliSeconds = rate / 1000;
         auto milliSecondsToWait = std::max(0., (double)size / rateByMilliSeconds - operationTimeMs);