Introduce BlobDepotDeleteQueue logic KIKIMR-19453

10 files changed, 272 insertions, 46 deletions

diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp
index 92fb6f78e8a..074df2071c0 100644
--- a/ydb/core/mind/bscontroller/config.cpp
+++ b/ydb/core/mind/bscontroller/config.cpp
@@ -343,10 +343,21 @@ namespace NKikimr::NBsController {
                 state.PDisks.DeleteExistingEntry(pdiskId);
             }
 
-            MakeTableMerger<Schema::HostConfig>(&HostConfigs, &state.HostConfigs.Get(), this)(txc);
-            MakeTableMerger<Schema::Box>(&Boxes, &state.Boxes.Get(), this)(txc);
-            MakeTableMerger<Schema::BoxStoragePool>(&StoragePools, &state.StoragePools.Get(), this)(txc);
-            MakeTableMerger<Schema::Node>(&Nodes, &state.Nodes.Get(), this)(txc);
+            if (state.HostConfigs.Changed()) {
+                MakeTableMerger<Schema::HostConfig>(&HostConfigs, &state.HostConfigs.Get(), this)(txc);
+            }
+            if (state.Boxes.Changed()) {
+                MakeTableMerger<Schema::Box>(&Boxes, &state.Boxes.Get(), this)(txc);
+            }
+            if (state.StoragePools.Changed()) {
+                MakeTableMerger<Schema::BoxStoragePool>(&StoragePools, &state.StoragePools.Get(), this)(txc);
+            }
+            if (state.Nodes.Changed()) {
+                MakeTableMerger<Schema::Node>(&Nodes, &state.Nodes.Get(), this)(txc);
+            }
+            if (state.BlobDepotDeleteQueue.Changed()) {
+                MakeTableMerger<Schema::BlobDepotDeleteQueue>(&BlobDepotDeleteQueue, &state.BlobDepotDeleteQueue.Get(), this)(txc);
+            }
 
             // apply overlay maps to their respective tables
             state.PDisks.ApplyToTable(this, txc);
diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h
index aa3d398e593..c31ebc66ad2 100644
--- a/ydb/core/mind/bscontroller/config.h
+++ b/ydb/core/mind/bscontroller/config.h
@@ -59,6 +59,7 @@ namespace NKikimr {
             TCowHolder<TMap<TBoxId, TBoxInfo>> Boxes;
             TCowHolder<TMap<TBoxStoragePoolId, TStoragePoolInfo>> StoragePools;
             TCowHolder<TMultiMap<TBoxStoragePoolId, TGroupId>> StoragePoolGroups;
+            TCowHolder<TMap<TGroupId, TBlobDepotDeleteQueueInfo>> BlobDepotDeleteQueue;
 
             // system-level configuration
             TOverlayMap<TPDiskId, TPDiskInfo> PDisks;
@@ -123,6 +124,7 @@ namespace NKikimr {
                 , Boxes(&controller.Boxes)
                 , StoragePools(&controller.StoragePools)
                 , StoragePoolGroups(&controller.StoragePoolGroups)
+                , BlobDepotDeleteQueue(&controller.BlobDepotDeleteQueue)
                 , PDisks(controller.PDisks)
                 , DrivesSerials(controller.DrivesSerials)
                 , Nodes(&controller.Nodes)
@@ -151,6 +153,7 @@ namespace NKikimr {
                 Boxes.Commit();
                 StoragePools.Commit();
                 StoragePoolGroups.Commit();
+                BlobDepotDeleteQueue.Commit();
                 PDisks.Commit();
                 DrivesSerials.Commit();
                 Nodes.Commit();
@@ -171,9 +174,10 @@ namespace NKikimr {
 
             bool Changed() const {
                 return HostConfigs.Changed() || Boxes.Changed() || StoragePools.Changed() ||
-                    StoragePoolGroups.Changed() || PDisks.Changed() || DrivesSerials.Changed() || Nodes.Changed() ||
-                    VSlots.Changed() || Groups.Changed() || IndexGroupSpeciesToGroup.Changed() || NextGroupId.Changed() ||
-                    NextStoragePoolId.Changed() || SerialManagementStage.Changed() || NextVirtualGroupId.Changed();
+                    StoragePoolGroups.Changed() || BlobDepotDeleteQueue.Changed() || PDisks.Changed() ||
+                    DrivesSerials.Changed() || Nodes.Changed() || VSlots.Changed() || Groups.Changed() ||
+                    IndexGroupSpeciesToGroup.Changed() || NextGroupId.Changed() || NextStoragePoolId.Changed() ||
+                    SerialManagementStage.Changed() || NextVirtualGroupId.Changed();
             }
 
             bool NormalizeHostKey(NKikimrBlobStorage::THostKey *host) const {
@@ -240,6 +244,20 @@ namespace NKikimr {
             }
 
             void DeleteExistingGroup(TGroupId groupId) {
+                const TGroupInfo *group = Groups.Find(groupId);
+                Y_VERIFY(group);
+                if (group->VirtualGroupState) { // this was a BlobDepot-based group, enqueue BlobDepot for deletion
+                    // parse blob depot config to figure out whether hive was contacted; if not, skip the HiveId field
+                    Y_VERIFY(group->BlobDepotConfig);
+                    NKikimrBlobDepot::TBlobDepotConfig config;
+                    const bool success = config.ParseFromString(*group->BlobDepotConfig);
+                    Y_VERIFY(success);
+                    if (config.GetHiveContacted()) {
+                        const auto [it, inserted] = BlobDepotDeleteQueue.Unshare().try_emplace(groupId, group->HiveId,
+                            config.HasTabletId() ? MakeMaybe(config.GetTabletId()) : Nothing());
+                        Y_VERIFY(inserted);
+                    }
+                }
                 Groups.DeleteExistingEntry(groupId);
                 GroupContentChanged.erase(groupId);
                 GroupFailureModelChanged.erase(groupId);
diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h
index 57ccd2b9dbb..cc74d434624 100644
--- a/ydb/core/mind/bscontroller/impl.h
+++ b/ydb/core/mind/bscontroller/impl.h
@@ -1369,6 +1369,33 @@ public:
         void OnClone(const THolder<TDriveSerialInfo>&) {}
     };
 
+    struct TBlobDepotDeleteQueueInfo {
+        using Table = Schema::BlobDepotDeleteQueue;
+
+        TMaybe<Table::HiveId::Type> HiveId;
+        TMaybe<Table::BlobDepotId::Type> BlobDepotId;
+        TActorId VirtualGroupSetupMachineId;
+
+        TBlobDepotDeleteQueueInfo() = default;
+
+        TBlobDepotDeleteQueueInfo(TMaybe<Table::HiveId::Type> hiveId, TMaybe<Table::BlobDepotId::Type> blobDepotId)
+            : HiveId(std::move(hiveId))
+            , BlobDepotId(std::move(blobDepotId))
+        {}
+
+        template<typename T>
+        static void Apply(TBlobStorageController* /*controller*/, T&& callback) {
+            static TTableAdapter<Table, TBlobDepotDeleteQueueInfo,
+                    Table::HiveId,
+                    Table::BlobDepotId
+                > adapter(
+                    &TBlobDepotDeleteQueueInfo::HiveId,
+                    &TBlobDepotDeleteQueueInfo::BlobDepotId
+                );
+            callback(&adapter);
+        }
+    };
+
     struct THostRecord {
         TNodeId NodeId;
         TNodeLocation Location;
@@ -1459,6 +1486,7 @@ private:
     TMap<TBoxId, TBoxInfo> Boxes;
     TMap<TBoxStoragePoolId, TStoragePoolInfo> StoragePools;
     TMultiMap<TBoxStoragePoolId, TGroupId> StoragePoolGroups;
+    TMap<TGroupId, TBlobDepotDeleteQueueInfo> BlobDepotDeleteQueue;
     ui64 NextOperationLogIndex = 1;
     TActorId StatProcessorActorId;
     TInstant LastMetricsCommit;
@@ -1681,6 +1709,11 @@ private:
                 TActivationContext::Send(new IEventHandle(TEvents::TSystem::Poison, 0, actorId, SelfId(), nullptr, 0));
             }
         }
+        for (const auto& [groupId, info] : BlobDepotDeleteQueue) {
+            if (const auto& actorId = info.VirtualGroupSetupMachineId) {
+                TActivationContext::Send(new IEventHandle(TEvents::TSystem::Poison, 0, actorId, SelfId(), nullptr, 0));
+            }
+        }
         TActivationContext::Send(new IEventHandle(TEvents::TSystem::Unsubscribe, 0, GetNameserviceActorId(), SelfId(),
             nullptr, 0));
         TActivationContext::Send(new IEventHandle(TEvents::TSystem::Unsubscribe, 0, MakeBlobStorageNodeWardenID(SelfId().NodeId()),
@@ -1790,6 +1823,7 @@ private:
         TScrubState(TBlobStorageController *self);
         ~TScrubState();
         void HandleTimer();
+        void Clear();
         void AddItem(TVSlotId vslotId, std::optional<TString> state, TInstant scrubCycleStartTime,
             TInstant scrubCycleFinishTime, std::optional<bool> success);
         void OnDeletePDisk(TPDiskId pdiskId);
@@ -2045,6 +2079,9 @@ public:
                 StartVirtualGroupSetupMachine(info.Get());
             }
         }
+        for (auto& [groupId, info] : BlobDepotDeleteQueue) {
+            StartVirtualGroupDeleteMachine(groupId, info);
+        }
 
         for (; !InitQueue.empty(); InitQueue.pop_front()) {
             TAutoPtr<IEventHandle> &ev = InitQueue.front();
@@ -2097,6 +2134,7 @@ public:
     void CommitVirtualGroupUpdates(TConfigState& state);
 
     void StartVirtualGroupSetupMachine(TGroupInfo *group);
+    void StartVirtualGroupDeleteMachine(ui32 groupId, TBlobDepotDeleteQueueInfo& info);
 
     void Handle(TEvBlobStorage::TEvControllerGroupDecommittedNotify::TPtr ev);
 
diff --git a/ydb/core/mind/bscontroller/load_everything.cpp b/ydb/core/mind/bscontroller/load_everything.cpp
index f8e49dff93b..334b7018cc4 100644
--- a/ydb/core/mind/bscontroller/load_everything.cpp
+++ b/ydb/core/mind/bscontroller/load_everything.cpp
@@ -37,6 +37,7 @@ public:
             auto groupLatencies = db.Table<Schema::GroupLatencies>().Select();
             auto scrubState = db.Table<Schema::ScrubState>().Select();
             auto pdiskSerial = db.Table<Schema::DriveSerial>().Select();
+            auto blobDepotDeleteQueue = db.Table<Schema::BlobDepotDeleteQueue>().Select();
             if (!state.IsReady()
                     || !nodes.IsReady()
                     || !disk.IsReady()
@@ -55,7 +56,8 @@ public:
                     || !groupStoragePool.IsReady()
                     || !groupLatencies.IsReady()
                     || !scrubState.IsReady()
-                    || !pdiskSerial.IsReady()) {
+                    || !pdiskSerial.IsReady()
+                    || !blobDepotDeleteQueue.IsReady()) {
                 return false;
             }
         }
@@ -237,7 +239,8 @@ public:
         if (!NTableAdapter::FetchTable<Schema::HostConfig>(db, Self, Self->HostConfigs)
                 || !NTableAdapter::FetchTable<Schema::Box>(db, Self, Self->Boxes)
                 || !NTableAdapter::FetchTable<Schema::BoxStoragePool>(db, Self, Self->StoragePools)
-                || !NTableAdapter::FetchTable<Schema::DriveSerial>(db, Self, Self->DrivesSerials)) {
+                || !NTableAdapter::FetchTable<Schema::DriveSerial>(db, Self, Self->DrivesSerials)
+                || !NTableAdapter::FetchTable<Schema::BlobDepotDeleteQueue>(db, Self, Self->BlobDepotDeleteQueue)) {
             return false;
         }
         for (const auto& [storagePoolId, storagePool] : Self->StoragePools) {
@@ -472,6 +475,7 @@ public:
         }
 
         // scrub state
+        Self->ScrubState.Clear();
         {
             using Table = Schema::ScrubState;
             auto scrubState = db.Table<Table>().Select();
diff --git a/ydb/core/mind/bscontroller/migrate.cpp b/ydb/core/mind/bscontroller/migrate.cpp
index adc66684d33..af7b589fd87 100644
--- a/ydb/core/mind/bscontroller/migrate.cpp
+++ b/ydb/core/mind/bscontroller/migrate.cpp
@@ -236,7 +236,7 @@ public:
     void Complete(const TActorContext& ctx) override {
         STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXM02, "Complete tx", (IncompatibleData, IncompatibleData));
         if (IncompatibleData) {
-            STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXM03, "CompatibilityInfo check failed", (ErrorReason, CompatibilityError));
+            STLOG(PRI_ALERT, BS_CONTROLLER, BSCTXM00, "CompatibilityInfo check failed", (ErrorReason, CompatibilityError));
             ctx.Send(new IEventHandle(TEvents::TSystem::Poison, 0, Self->SelfId(), {}, nullptr, 0));
         } else {
             Self->Execute(new TTxQueue(Self, std::move(Queue)));
diff --git a/ydb/core/mind/bscontroller/scheme.h b/ydb/core/mind/bscontroller/scheme.h
index e1b0d068dcd..c28dc70636e 100644
--- a/ydb/core/mind/bscontroller/scheme.h
+++ b/ydb/core/mind/bscontroller/scheme.h
@@ -409,35 +409,45 @@ struct Schema : NIceDb::Schema {
 
     // struct VirtualGroupPool : Table<130> {};
 
+    struct BlobDepotDeleteQueue : Table<131> {
+        struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {}; // PK
+        struct HiveId : Column<2, NScheme::NTypeIds::Uint64> {};
+        struct BlobDepotId : Column<3, NScheme::NTypeIds::Uint64> {}; // may be null if creation wasn't confirmed
+
+        using TKey = TableKey<GroupId>;
+        using TColumns = TableColumns<GroupId, HiveId, BlobDepotId>;
+    };
+
     using TTables = SchemaTables<
-                                Node,
-                                PDisk,
-                                Group,
-                                State,
-                                VSlot,
-                                VDiskMetrics,
-                                PDiskMetrics,
-                                GroupLatencies,
-                                Box,
-                                BoxUser,
-                                HostConfig,
-                                HostConfigDrive,
-                                BoxHostV2,
-                                BoxStoragePool,
-                                BoxStoragePoolUser,
-                                BoxStoragePoolPDiskFilter,
-                                GroupStoragePool,
-                                OperationLog,
-                                MigrationPlan,
-                                MigrationEntry,
-                                ScrubState,
-                                DriveSerial
-                                >;
+        Node,
+        PDisk,
+        Group,
+        State,
+        VSlot,
+        VDiskMetrics,
+        PDiskMetrics,
+        GroupLatencies,
+        Box,
+        BoxUser,
+        HostConfig,
+        HostConfigDrive,
+        BoxHostV2,
+        BoxStoragePool,
+        BoxStoragePoolUser,
+        BoxStoragePoolPDiskFilter,
+        GroupStoragePool,
+        OperationLog,
+        MigrationPlan,
+        MigrationEntry,
+        ScrubState,
+        DriveSerial,
+        BlobDepotDeleteQueue
+    >;
 
     using TSettings = SchemaSettings<
-                                    ExecutorLogBatching<true>,
-                                    ExecutorLogFlushPeriod<TDuration::MicroSeconds(512).GetValue()>
-                                    >;
+        ExecutorLogBatching<true>,
+        ExecutorLogFlushPeriod<TDuration::MicroSeconds(512).GetValue()>
+    >;
 };
 
 } // NBsController
diff --git a/ydb/core/mind/bscontroller/scrub.cpp b/ydb/core/mind/bscontroller/scrub.cpp
index 4bf46624dff..8d3d7a638f3 100644
--- a/ydb/core/mind/bscontroller/scrub.cpp
+++ b/ydb/core/mind/bscontroller/scrub.cpp
@@ -144,6 +144,8 @@ class TBlobStorageController::TScrubState::TImpl {
         Y_FAIL();
     }
 
+    friend class TBlobStorageController;
+
 public:
     TImpl(TBlobStorageController *self)
         : Self(self)
@@ -780,6 +782,10 @@ void TBlobStorageController::TScrubState::HandleTimer() {
     TActivationContext::Schedule(TDuration::Minutes(1), new IEventHandle(Impl->SelfId(), {}, new TEvPrivate::TEvScrub));
 }
 
+void TBlobStorageController::TScrubState::Clear() {
+    Impl.reset(new TImpl(Impl->Self));
+}
+
 void TBlobStorageController::TScrubState::AddItem(TVSlotId vslotId, std::optional<TString> state,
         TInstant scrubCycleStartTime, TInstant scrubCycleFinishTime, std::optional<bool> success) {
     Impl->AddItem(vslotId, std::move(state), scrubCycleStartTime, scrubCycleFinishTime, success);
diff --git a/ydb/core/mind/bscontroller/virtual_group.cpp b/ydb/core/mind/bscontroller/virtual_group.cpp
index 03cb7ab4ef3..31fdee14c60 100644
--- a/ydb/core/mind/bscontroller/virtual_group.cpp
+++ b/ydb/core/mind/bscontroller/virtual_group.cpp
@@ -216,6 +216,7 @@ namespace NKikimr::NBsController {
         TBlobStorageController *Self;
         const TActorId ControllerId;
         const TGroupId GroupId;
+        const std::optional<TBlobDepotDeleteQueueInfo> DeleteInfo;
 
     private:
         class TTxUpdateGroup : public TTransactionBase<TBlobStorageController> {
@@ -269,6 +270,48 @@ namespace NKikimr::NBsController {
             }
         };
 
+        class TTxDeleteBlobDepot : public TTransactionBase<TBlobStorageController> {
+            TVirtualGroupSetupMachine* const Machine;
+            const TActorId MachineId;
+            const TGroupId GroupId;
+            const std::weak_ptr<TToken> Token;
+            std::optional<TConfigState> State;
+
+        public:
+            TTxType GetTxType() const override { return NBlobStorageController::TXTYPE_DELETE_BLOB_DEPOT; }
+
+            TTxDeleteBlobDepot(TVirtualGroupSetupMachine *machine)
+                : TTransactionBase(machine->Self)
+                , Machine(machine)
+                , MachineId(Machine->SelfId())
+                , GroupId(Machine->GroupId)
+                , Token(Machine->Token)
+            {}
+
+            bool Execute(TTransactionContext& txc, const TActorContext&) override {
+                if (Token.expired()) {
+                    return true; // actor is already dead
+                }
+                State.emplace(*Self, Self->HostRecords, TActivationContext::Now());
+                const size_t n = State->BlobDepotDeleteQueue.Unshare().erase(GroupId);
+                Y_VERIFY(n == 1);
+                TString error;
+                if (State->Changed() && !Self->CommitConfigUpdates(*State, true, true, true, txc, &error)) {
+                    STLOG(PRI_ERROR, BS_CONTROLLER, BSCVG17, "failed to commit update", (VirtualGroupId, GroupId), (Error, error));
+                    State->Rollback();
+                    State.reset();
+                }
+                return true;
+            }
+
+            void Complete(const TActorContext&) override {
+                if (State) {
+                    State->ApplyConfigUpdates();
+                }
+                TActivationContext::Send(new IEventHandle(TEvents::TSystem::Bootstrap, 0, MachineId, {}, nullptr, 0));
+            }
+        };
+
     public:
         TVirtualGroupSetupMachine(TBlobStorageController *self, TGroupInfo& group)
             : Self(self)
@@ -276,12 +319,31 @@ namespace NKikimr::NBsController {
             , GroupId(group.ID)
         {}
 
+        TVirtualGroupSetupMachine(TBlobStorageController *self, ui32 groupId, const TBlobDepotDeleteQueueInfo& info)
+            : Self(self)
+            , ControllerId(Self->SelfId())
+            , GroupId(groupId)
+            , DeleteInfo(info)
+        {}
+
         void Bootstrap() {
             Become(&TThis::StateFunc);
             if (Expired()) { // BS_CONTROLLER is already dead
                 return PassAway();
             }
 
+            if (DeleteInfo) {
+                Y_VERIFY(Self->BlobDepotDeleteQueue.contains(GroupId));
+                STLOG(PRI_DEBUG, BS_CONTROLLER, BSCVG19, "Bootstrap for delete", (GroupId, GroupId),
+                    (HiveId, DeleteInfo->HiveId), (BlobDepotId, DeleteInfo->BlobDepotId));
+                if (DeleteInfo->HiveId) {
+                    HiveDelete(*DeleteInfo->HiveId, DeleteInfo->BlobDepotId);
+                } else {
+                    OnBlobDepotDeleted();
+                }
+                return;
+            }
+
             TGroupInfo *group = GetGroup();
             if (!group->VirtualGroupState) { // group was deleted or reset to non-decommitting during the last transaction
                 return PassAway();
@@ -514,16 +576,20 @@ namespace NKikimr::NBsController {
                 return DeleteBlobDepot();
             }
 
+            HiveDelete(*group->HiveId, config.HasTabletId() ? MakeMaybe(config.GetTabletId()) : Nothing());
+        }
+
+        void HiveDelete(ui64 hiveId, TMaybe<ui64> tabletId) {
             Y_VERIFY(!HivePipeId);
-            Y_VERIFY(group->HiveId);
-            HivePipeId = Register(NTabletPipe::CreateClient(SelfId(), *group->HiveId, NTabletPipe::TClientRetryPolicy::WithRetries()));
+            Y_VERIFY(hiveId);
+            HivePipeId = Register(NTabletPipe::CreateClient(SelfId(), hiveId, NTabletPipe::TClientRetryPolicy::WithRetries()));
 
-            auto ev = config.HasTabletId()
-                ? std::make_unique<TEvHive::TEvDeleteTablet>(Self->TabletID(), group->ID, config.GetTabletId(), 0)
-                : std::make_unique<TEvHive::TEvDeleteTablet>(Self->TabletID(), group->ID, 0);
+            auto ev = tabletId
+                ? std::make_unique<TEvHive::TEvDeleteTablet>(Self->TabletID(), GroupId, *tabletId, 0)
+                : std::make_unique<TEvHive::TEvDeleteTablet>(Self->TabletID(), GroupId, 0);
 
-            STLOG(PRI_INFO, BS_CONTROLLER, BSCVG12, "sending TEvDeleteTablet", (GroupId, group->ID),
-                (HiveId, *group->HiveId), (Msg, ev->Record));
+            STLOG(PRI_INFO, BS_CONTROLLER, BSCVG12, "sending TEvDeleteTablet", (GroupId, GroupId),
+                (HiveId, hiveId), (Msg, ev->Record));
 
             NTabletPipe::SendData(SelfId(), HivePipeId, ev.release());
         }
@@ -535,7 +601,7 @@ namespace NKikimr::NBsController {
 
             if (ev->Get()->Status != NKikimrProto::OK) {
                 OnPipeError(ev->Get()->ClientId);
-            } else if (ev->Get()->ClientId == HivePipeId) {
+            } else if (ev->Get()->ClientId == HivePipeId && !DeleteInfo) {
                 TGroupInfo *group = GetGroup();
                 if (group->VirtualGroupState == NKikimrBlobStorage::EVirtualGroupState::NEW) {
                     auto& config = GetConfig(group);
@@ -618,7 +684,11 @@ namespace NKikimr::NBsController {
         void Handle(TEvHive::TEvDeleteTabletReply::TPtr ev) {
             STLOG(PRI_INFO, BS_CONTROLLER, BSCVG13, "received TEvDeleteTabletReply", (GroupId, GroupId),
                 (Msg, ev->Get()->Record));
-            DeleteBlobDepot();
+            if (DeleteInfo) {
+                OnBlobDepotDeleted();
+            } else {
+                DeleteBlobDepot();
+            }
         }
 
         void ConfigureBlobDepot() {
@@ -654,6 +724,11 @@ namespace NKikimr::NBsController {
             }));
         }
 
+        void OnBlobDepotDeleted() {
+            STLOG(PRI_DEBUG, BS_CONTROLLER, BSCVG18, "OnBlobDepotDeleted", (GroupId, GroupId));
+            Self->Execute(std::make_unique<TTxDeleteBlobDepot>(this));
+        }
+
         void Handle(TEvBlobDepot::TEvApplyConfigResult::TPtr /*ev*/) {
             NTabletPipe::CloseAndForgetClient(SelfId(), BlobDepotPipeId);
 
@@ -688,7 +763,7 @@ namespace NKikimr::NBsController {
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
         void PassAway() override {
-            if (!Expired()) {
+            if (!Expired() && !DeleteInfo) {
                 TGroupInfo *group = GetGroup();
                 group->VirtualGroupSetupMachineId = {};
             }
@@ -723,6 +798,7 @@ namespace NKikimr::NBsController {
         }
 
         TGroupInfo *GetGroup() {
+            Y_VERIFY(!DeleteInfo);
             TGroupInfo *res = Self->FindGroup(GroupId);
             Y_VERIFY(res);
             return res;
@@ -731,6 +807,8 @@ namespace NKikimr::NBsController {
         bool Expired() const {
             if (!TlsActivationContext->Mailbox.FindActor(ControllerId.LocalId())) { // BS_CONTROLLER died
                 return true;
+            } else if (DeleteInfo) {
+                return !Self->BlobDepotDeleteQueue.contains(GroupId);
             } else if (const TGroupInfo *group = Self->FindGroup(GroupId); !group) { // group is deleted
                 return true;
             } else if (group->VirtualGroupSetupMachineId != SelfId()) { // another machine is started
@@ -773,6 +851,14 @@ namespace NKikimr::NBsController {
                 startSetupMachine(restartNeeded);
             }
         }
+
+        if (state.BlobDepotDeleteQueue.Changed()) {
+            for (const auto& [prev, cur] : Diff(&BlobDepotDeleteQueue, &state.BlobDepotDeleteQueue.Unshare())) {
+                if (!prev) { // a new item was just inserted, start delete machine
+                    StartVirtualGroupDeleteMachine(cur->first, cur->second);
+                }
+            }
+        }
     }
 
     void TBlobStorageController::StartVirtualGroupSetupMachine(TGroupInfo *group) {
@@ -780,6 +866,11 @@ namespace NKikimr::NBsController {
         group->VirtualGroupSetupMachineId = RegisterWithSameMailbox(new TVirtualGroupSetupMachine(this, *group));
     }
 
+    void TBlobStorageController::StartVirtualGroupDeleteMachine(ui32 groupId, TBlobDepotDeleteQueueInfo& info) {
+        Y_VERIFY(!info.VirtualGroupSetupMachineId);
+        info.VirtualGroupSetupMachineId = RegisterWithSameMailbox(new TVirtualGroupSetupMachine(this, groupId, info));
+    }
+
     void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerGroupDecommittedNotify::TPtr ev) {
         class TTxDecommitGroup : public TTransactionBase<TBlobStorageController> {
             TEvBlobStorage::TEvControllerGroupDecommittedNotify::TPtr Ev;
diff --git a/ydb/core/protos/counters_bs_controller.proto b/ydb/core/protos/counters_bs_controller.proto
index 0a74ca8ab7e..8d5c193fdf1 100644
--- a/ydb/core/protos/counters_bs_controller.proto
+++ b/ydb/core/protos/counters_bs_controller.proto
@@ -274,4 +274,5 @@ enum ETxTypes {
     TXTYPE_GROUP_METRICS_EXCHANGE = 25 [(TxTypeOpts) = {Name: "TTxGroupMetricsExchange"}];
     TXTYPE_DECOMMIT_GROUP = 26 [(TxTypeOpts) = {Name: "TTxDecommitGroup"}];
     TXTYPE_UPDATE_GROUP = 27 [(TxTypeOpts) = {Name: "TTxUpdateGroup"}];
+    TXTYPE_DELETE_BLOB_DEPOT = 28 [(TxTypeOpts) = {Name: "TTxDeleteBlobDepot"}];
 }
diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema
index 93074d50a8d..1a12c9bd114 100644
--- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema
+++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema
@@ -1573,6 +1573,53 @@
         }
     },
     {
+        "TableId": 131,
+        "TableName": "BlobDepotDeleteQueue",
+        "TableKey": [
+            1
+        ],
+        "ColumnsAdded": [
+            {
+                "ColumnId": 1,
+                "ColumnName": "GroupId",
+                "ColumnType": "Uint32"
+            },
+            {
+                "ColumnId": 2,
+                "ColumnName": "HiveId",
+                "ColumnType": "Uint64"
+            },
+            {
+                "ColumnId": 3,
+                "ColumnName": "BlobDepotId",
+                "ColumnType": "Uint64"
+            }
+        ],
+        "ColumnsDropped": [],
+        "ColumnFamilies": {
+            "0": {
+                "Columns": [
+                    1,
+                    2,
+                    3
+                ],
+                "RoomID": 0,
+                "Codec": 0,
+                "InMemory": false,
+                "Cache": 0,
+                "Small": 4294967295,
+                "Large": 4294967295
+            }
+        },
+        "Rooms": {
+            "0": {
+                "Main": 1,
+                "Outer": 1,
+                "Blobs": 1
+            }
+        }
+    },
+    {
         "TableId": 102,
         "TableName": "HostConfig",
         "TableKey": [