summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov333 <[email protected]>2025-07-10 14:59:12 +0300
committerGitHub <[email protected]>2025-07-10 14:59:12 +0300
commit666035fb9411bf8f58bfdba3a24cf82fed7eed37 (patch)
tree91f8a735a26c5d6416a4d55b51083ec82b3a3106
parent6e61d6058a51532b7fae1fa03aa3e6479e3dc5ba (diff)
schemas sysview (#20917)
-rw-r--r--ydb/core/kqp/ut/olap/write_ut.cpp3
-rw-r--r--ydb/core/sys_view/common/schema.cpp2
-rw-r--r--ydb/core/sys_view/common/schema.h22
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp40
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h107
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp32
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h36
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp59
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h33
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp58
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h31
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make14
-rw-r--r--ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make1
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h5
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp9
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h3
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h4
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h17
-rw-r--r--ydb/services/ydb/ydb_logstore_ut.cpp10
19 files changed, 477 insertions, 9 deletions
diff --git a/ydb/core/kqp/ut/olap/write_ut.cpp b/ydb/core/kqp/ut/olap/write_ut.cpp
index 60739b07689..5d3f16282cd 100644
--- a/ydb/core/kqp/ut/olap/write_ut.cpp
+++ b/ydb/core/kqp/ut/olap/write_ut.cpp
@@ -185,6 +185,9 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
READ: SELECT Col1, Col4, Col5, Col7, Col8 FROM `/Root/ColumnTable` ORDER BY Col1;
EXPECTED: [[1u;#;#;#;#];[2u;#;#;#;#];[3u;#;#;#;#];[4u;[4u];#;#;#];[5u;[5u];[5u];#;#];[6u;[6u];[6u];#;#];[7u;[7u];[7u];[7u];#];[8u;[8u];[8u];[8u];[8u]];[9u;[9u];[9u];[9u];[9u]]]
------
+ READ: SELECT SchemaVersion FROM `/Root/ColumnTable/.sys/primary_index_schema_stats` WHERE PresetId = 0 ORDER BY SchemaVersion;
+ EXPECTED: [[[6u]];[[9u]];[[11u]]]
+ ------
DATA:
DELETE FROM `/Root/ColumnTable`
------
diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/schema.cpp
index 51cb307bee5..4462c0abec7 100644
--- a/ydb/core/sys_view/common/schema.cpp
+++ b/ydb/core/sys_view/common/schema.cpp
@@ -319,10 +319,12 @@ private:
RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName, ESysViewType::EQueryMetricsOneMinute);
RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName);
+ RegisterOlapStoreSystemView<Schema::PrimaryIndexSchemaStats>(StorePrimaryIndexSchemaStatsName);
RegisterOlapStoreSystemView<Schema::PrimaryIndexPortionStats>(StorePrimaryIndexPortionStatsName);
RegisterOlapStoreSystemView<Schema::PrimaryIndexGranuleStats>(StorePrimaryIndexGranuleStatsName);
RegisterOlapStoreSystemView<Schema::PrimaryIndexOptimizerStats>(StorePrimaryIndexOptimizerStatsName);
RegisterColumnTableSystemView<Schema::PrimaryIndexStats>(TablePrimaryIndexStatsName);
+ RegisterColumnTableSystemView<Schema::PrimaryIndexSchemaStats>(TablePrimaryIndexSchemaStatsName);
RegisterColumnTableSystemView<Schema::PrimaryIndexPortionStats>(TablePrimaryIndexPortionStatsName);
RegisterColumnTableSystemView<Schema::PrimaryIndexGranuleStats>(TablePrimaryIndexGranuleStatsName);
RegisterColumnTableSystemView<Schema::PrimaryIndexOptimizerStats>(TablePrimaryIndexOptimizerStatsName);
diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h
index f26af071cc7..b23d27a6145 100644
--- a/ydb/core/sys_view/common/schema.h
+++ b/ydb/core/sys_view/common/schema.h
@@ -36,10 +36,12 @@ constexpr TStringBuf TabletsName = "hive_tablets";
constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute";
constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats";
+constexpr TStringBuf StorePrimaryIndexSchemaStatsName = "store_primary_index_schema_stats";
constexpr TStringBuf StorePrimaryIndexPortionStatsName = "store_primary_index_portion_stats";
constexpr TStringBuf StorePrimaryIndexGranuleStatsName = "store_primary_index_granule_stats";
constexpr TStringBuf StorePrimaryIndexOptimizerStatsName = "store_primary_index_optimizer_stats";
constexpr TStringBuf TablePrimaryIndexStatsName = "primary_index_stats";
+constexpr TStringBuf TablePrimaryIndexSchemaStatsName = "primary_index_schema_stats";
constexpr TStringBuf TablePrimaryIndexPortionStatsName = "primary_index_portion_stats";
constexpr TStringBuf TablePrimaryIndexGranuleStatsName = "primary_index_granule_stats";
constexpr TStringBuf TablePrimaryIndexOptimizerStatsName = "primary_index_optimizer_stats";
@@ -804,6 +806,26 @@ struct Schema : NIceDb::Schema {
IndexSize,
FollowerId>;
};
+
+ struct PrimaryIndexSchemaStats : Table<24> {
+ struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct PresetId : Column<2, NScheme::NTypeIds::Uint64> {};
+ struct SchemaVersion : Column<3, NScheme::NTypeIds::Uint64> {};
+ struct SchemaSnapshotPlanStep : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct SchemaSnapshotTxId : Column<5, NScheme::NTypeIds::Uint64> {};
+ struct SchemaDetails : Column<6, NScheme::NTypeIds::Utf8> {};
+
+ using TKey = TableKey<TabletId, PresetId, SchemaVersion>;
+ using TColumns = TableColumns<
+ TabletId,
+ PresetId,
+ SchemaVersion,
+ SchemaSnapshotPlanStep,
+ SchemaSnapshotTxId,
+ SchemaDetails
+ >;
+ };
+
};
bool MaybeSystemViewPath(const TVector<TString>& path);
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp
new file mode 100644
index 00000000000..626b2a405f3
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp
@@ -0,0 +1,40 @@
+#include "constructor.h"
+
+#include <ydb/core/tx/columnshard/engines/column_engine_logs.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+
+TConstructor::TConstructor(
+ const IColumnEngine& engine, const ui64 tabletId, const std::shared_ptr<NOlap::TPKRangesFilter>& pkFilter, const bool isReverseSort)
+ : TabletId(tabletId) {
+ const TColumnEngineForLogs* engineImpl = dynamic_cast<const TColumnEngineForLogs*>(&engine);
+ std::vector<ISnapshotSchema::TPtr> schemasAll;
+ for (auto&& i : engineImpl->GetVersionedSchemas().GetPresetVersionedIndex()) {
+ for (auto&& [_, s] : i.second->GetSnapshotByVersions()) {
+ schemasAll.emplace_back(s);
+ }
+ }
+ const auto pred = [](const ISnapshotSchema::TPtr& l, const ISnapshotSchema::TPtr& r) {
+ return std::tuple(l->GetIndexInfo().GetPresetId(), l->GetVersion()) < std::tuple(r->GetIndexInfo().GetPresetId(), r->GetVersion());
+ };
+ std::sort(schemasAll.begin(), schemasAll.end(), pred);
+ std::vector<ISnapshotSchema::TPtr> current;
+ for (auto&& i : schemasAll) {
+ current.emplace_back(i);
+ if (current.size() == 10) {
+ AddConstructors(std::move(current), pkFilter);
+ current.clear();
+ }
+ }
+ if (current.size()) {
+ AddConstructors(std::move(current), pkFilter);
+ current.clear();
+ }
+
+ std::sort(Constructors.begin(), Constructors.end(), TDataConstructor::TComparator(isReverseSort));
+ for (ui32 idx = 0; idx < Constructors.size(); ++idx) {
+ Constructors[idx].SetIndex(idx);
+ }
+}
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h
new file mode 100644
index 00000000000..2055085a8c8
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h
@@ -0,0 +1,107 @@
+#pragma once
+#include "schema.h"
+#include "source.h"
+
+#include <ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/constructor.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+class TDataConstructor {
+private:
+ ui64 TabletId;
+ std::vector<ISnapshotSchema::TPtr> Schemas;
+ NArrow::TSimpleRow Start;
+ NArrow::TSimpleRow Finish;
+ ui32 SourceId = 0;
+ ui32 SourceIdx = 0;
+
+public:
+ void SetIndex(const ui32 index) {
+ AFL_VERIFY(!SourceId);
+ SourceIdx = index;
+ SourceId = index + 1;
+ }
+
+ TDataConstructor(const ui64 tabletId, std::vector<ISnapshotSchema::TPtr>&& schemas)
+ : TabletId(tabletId)
+ , Schemas(std::move(schemas))
+ , Start(TSchemaAdapter::GetPKSimpleRow(TabletId, Schemas.front()->GetIndexInfo().GetPresetId(), Schemas.front()->GetVersion()))
+ , Finish(TSchemaAdapter::GetPKSimpleRow(TabletId, Schemas.back()->GetIndexInfo().GetPresetId(), Schemas.back()->GetVersion())) {
+ if (Schemas.size() > 1) {
+ AFL_VERIFY(Start < Finish)("start", Start.DebugString())("finish", Finish.DebugString());
+ }
+ }
+
+ const NArrow::TSimpleRow& GetStart() const {
+ return Start;
+ }
+ const NArrow::TSimpleRow& GetFinish() const {
+ return Finish;
+ }
+
+ struct TComparator {
+ private:
+ const bool IsReverse;
+
+ public:
+ TComparator(const bool isReverse)
+ : IsReverse(isReverse) {
+ }
+
+ bool operator()(const TDataConstructor& l, const TDataConstructor& r) const {
+ if (IsReverse) {
+ return r.Finish < l.Finish;
+ } else {
+ return l.Start < r.Start;
+ }
+ }
+ };
+
+ std::shared_ptr<NReader::NSimple::IDataSource> Construct(const std::shared_ptr<NReader::NSimple::TSpecialReadContext>& context) {
+ AFL_VERIFY(SourceId);
+ return std::make_shared<TSourceData>(SourceId, SourceIdx, TabletId, std::move(Schemas), std::move(Start), std::move(Finish), context);
+ }
+};
+
+class TConstructor: public NAbstract::ISourcesConstructor {
+private:
+ const ui64 TabletId;
+ std::deque<TDataConstructor> Constructors;
+
+ virtual void DoClear() override {
+ Constructors.clear();
+ }
+ virtual void DoAbort() override {
+ Constructors.clear();
+ }
+ virtual bool DoIsFinished() const override {
+ return Constructors.empty();
+ }
+ virtual std::shared_ptr<NReader::NCommon::IDataSource> DoExtractNext(
+ const std::shared_ptr<NReader::NCommon::TSpecialReadContext>& context) override {
+ AFL_VERIFY(Constructors.size());
+ std::shared_ptr<NReader::NCommon::IDataSource> result =
+ Constructors.front().Construct(std::static_pointer_cast<NReader::NSimple::TSpecialReadContext>(context));
+ Constructors.pop_front();
+ return result;
+ }
+ virtual void DoInitCursor(const std::shared_ptr<IScanCursor>& /*cursor*/) override {
+ }
+ virtual TString DoDebugString() const override {
+ return Default<TString>();
+ }
+
+ void AddConstructors(std::vector<ISnapshotSchema::TPtr>&& schemas, const std::shared_ptr<NOlap::TPKRangesFilter>& pkFilter) {
+ Constructors.emplace_back(TabletId, std::move(schemas));
+ if (!pkFilter->IsUsed(Constructors.back().GetStart(), Constructors.back().GetFinish())) {
+ Constructors.pop_back();
+ }
+ }
+
+public:
+ TConstructor(
+ const IColumnEngine& engine, const ui64 tabletId, const std::shared_ptr<NOlap::TPKRangesFilter>& pkFilter, const bool isReverseSort);
+};
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp
new file mode 100644
index 00000000000..6a1386e2c20
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp
@@ -0,0 +1,32 @@
+#include "constructor.h"
+#include "metadata.h"
+#include "schema.h"
+
+#include <ydb/core/tx/columnshard/engines/reader/common/description.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+
+TAccessor::TAccessor(const TString& tablePath, const NColumnShard::TSchemeShardLocalPathId externalPathId,
+ const std::optional<NColumnShard::TInternalPathId> internalPathId)
+ : TBase(tablePath)
+ , PathId(NColumnShard::TUnifiedPathId::BuildNoCheck(internalPathId, externalPathId)) {
+ AFL_VERIFY(CheckTablePath(GetTablePath()));
+}
+
+std::unique_ptr<NReader::NCommon::ISourcesConstructor> TAccessor::SelectMetadata(const TSelectMetadataContext& context,
+ const NReader::TReadDescription& readDescription, const bool /*withUncommitted*/, const bool isPlain) const {
+ AFL_VERIFY(!isPlain);
+ return std::make_unique<TConstructor>(
+ context.GetEngine(), readDescription.GetTabletId(), readDescription.PKRangesFilter, readDescription.IsReverseSort());
+}
+
+std::shared_ptr<ISnapshotSchema> TAccessor::GetSnapshotSchemaOptional(
+ const TVersionedPresetSchemas& vSchemas, const TSnapshot& /*snapshot*/) const {
+ return vSchemas.GetVersionedIndex(TSchemaAdapter::GetInstance().GetPresetId()).GetLastSchema();
+}
+
+std::shared_ptr<const TVersionedIndex> TAccessor::GetVersionedIndexCopyOptional(TVersionedPresetSchemas& vSchemas) const {
+ return vSchemas.GetVersionedIndexCopy(TSchemaAdapter::GetInstance().GetPresetId());
+}
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h
new file mode 100644
index 00000000000..4a83ed44ec4
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h
@@ -0,0 +1,36 @@
+#pragma once
+#include <ydb/core/tx/columnshard/engines/metadata_accessor.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/metadata.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+
+class TAccessor: public NAbstract::TAccessor {
+private:
+ using TBase = NAbstract::TAccessor;
+ const NColumnShard::TUnifiedPathId PathId;
+
+ virtual std::optional<NColumnShard::TUnifiedPathId> GetPathId() const override {
+ return PathId;
+ }
+
+ virtual std::shared_ptr<const TVersionedIndex> GetVersionedIndexCopyOptional(TVersionedPresetSchemas& vSchemas) const override;
+
+ virtual std::shared_ptr<ISnapshotSchema> GetSnapshotSchemaOptional(
+ const TVersionedPresetSchemas& vSchemas, const TSnapshot& snapshot) const override;
+
+public:
+ static bool CheckTablePath(const TString& tablePath) {
+ return tablePath.EndsWith("/.sys/primary_index_schema_stats") || tablePath.EndsWith("/.sys/store_primary_index_schema_stats");
+ }
+
+ TAccessor(const TString& tableName, const NColumnShard::TSchemeShardLocalPathId externalPathId,
+ const std::optional<NColumnShard::TInternalPathId> internalPathId);
+ virtual std::unique_ptr<NReader::NCommon::ISourcesConstructor> SelectMetadata(const TSelectMetadataContext& context,
+ const NReader::TReadDescription& readDescription, const bool withUncommitted, const bool isPlain) const override;
+ virtual std::optional<TGranuleShardingInfo> GetShardingInfo(
+ const std::shared_ptr<const TVersionedIndex>& /*indexVersionsPointer*/, const NOlap::TSnapshot& /*ss*/) const override {
+ return std::nullopt;
+ }
+};
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp
new file mode 100644
index 00000000000..187664d52f2
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp
@@ -0,0 +1,59 @@
+#include "metadata.h"
+#include "schema.h"
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+
+NArrow::TSimpleRow TSchemaAdapter::GetPKSimpleRow(const ui64 tabletId, const ui64 presetId, const ui64 schemaVersion) {
+ NArrow::TSimpleRowViewV0::TWriter writer(sizeof(ui64) * 2);
+ writer.Append<ui64>(tabletId);
+ writer.Append<ui64>(presetId);
+ writer.Append<ui64>(schemaVersion);
+ return NArrow::TSimpleRow(writer.Finish(), GetPKSchema());
+}
+
+std::shared_ptr<arrow::Schema> TSchemaAdapter::GetPKSchema() {
+ static std::shared_ptr<arrow::Schema> schema = []() {
+ arrow::FieldVector fields = { std::make_shared<arrow::Field>("TabletId", arrow::uint64()),
+ std::make_shared<arrow::Field>("PresetId", arrow::uint64()), std::make_shared<arrow::Field>("SchemaVersion", arrow::uint64()) };
+ return std::make_shared<arrow::Schema>(std::move(fields));
+ }();
+ return schema;
+}
+
+TIndexInfo TSchemaAdapter::GetIndexInfo(
+ const std::shared_ptr<IStoragesManager>& storagesManager, const std::shared_ptr<TSchemaObjectsCache>& schemaObjectsCache) const {
+ //PrimaryIndexSchemaStats
+
+ static NKikimrSchemeOp::TColumnTableSchema proto = []() {
+ NKikimrSchemeOp::TColumnTableSchema proto;
+ ui32 currentId = 0;
+ const auto pred = [&](const TString& name, const NScheme::TTypeId typeId, const std::optional<ui32> entityId = std::nullopt) {
+ auto* col = proto.AddColumns();
+ col->SetId(entityId.value_or(++currentId));
+ col->SetName(name);
+ col->SetTypeId(typeId);
+ };
+ pred("TabletId", NScheme::NTypeIds::Uint64);
+ pred("PresetId", NScheme::NTypeIds::Uint64);
+ pred("SchemaVersion", NScheme::NTypeIds::Uint64);
+ pred("SchemaSnapshotPlanStep", NScheme::NTypeIds::Uint64);
+ pred("SchemaSnapshotTxId", NScheme::NTypeIds::Uint64);
+ pred("SchemaDetails", NScheme::NTypeIds::Utf8);
+
+ proto.AddKeyColumnNames("TabletId");
+ proto.AddKeyColumnNames("PresetId");
+ proto.AddKeyColumnNames("SchemaVersion");
+ return proto;
+ }();
+
+ auto indexInfo = TIndexInfo::BuildFromProto(GetPresetId(), proto, storagesManager, schemaObjectsCache);
+ AFL_VERIFY(indexInfo);
+ return std::move(*indexInfo);
+}
+
+std::shared_ptr<ITableMetadataAccessor> TSchemaAdapter::BuildMetadataAccessor(const TString& tableName,
+ const NColumnShard::TSchemeShardLocalPathId externalPathId, const std::optional<NColumnShard::TInternalPathId> internalPathId) const {
+ return std::make_shared<TAccessor>(tableName, externalPathId, internalPathId);
+}
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h
new file mode 100644
index 00000000000..fd974f53176
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h
@@ -0,0 +1,33 @@
+#pragma once
+#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
+#include <ydb/core/tx/columnshard/common/path_id.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/schema.h>
+#include <ydb/core/tx/columnshard/engines/scheme/index_info.h>
+#include <ydb/core/tx/columnshard/engines/scheme/objects_cache.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+
+class TSchemaAdapter: public NAbstract::ISchemaAdapter {
+private:
+ static const inline auto Registrator1 = TFactory::TRegistrator<TSchemaAdapter>("store_primary_index_schema_stats");
+ static const inline auto Registrator2 = TFactory::TRegistrator<TSchemaAdapter>("primary_index_schema_stats");
+
+public:
+ static const TSchemaAdapter& GetInstance() {
+ return *Singleton<TSchemaAdapter>();
+ }
+
+ virtual ui64 GetPresetId() const override {
+ static ui64 presetId = NAbstract::ISchemaAdapter::Counter.Inc();
+ return Max<ui64>() - presetId;
+ }
+ static NArrow::TSimpleRow GetPKSimpleRow(const ui64 tabletId, const ui64 presetId, const ui64 schemaVersion);
+ static std::shared_ptr<arrow::Schema> GetPKSchema();
+ virtual TIndexInfo GetIndexInfo(
+ const std::shared_ptr<IStoragesManager>& storagesManager, const std::shared_ptr<TSchemaObjectsCache>& schemaObjectsCache) const override;
+ virtual std::shared_ptr<ITableMetadataAccessor> BuildMetadataAccessor(const TString& tableName,
+ const NColumnShard::TSchemeShardLocalPathId externalPathId,
+ const std::optional<NColumnShard::TInternalPathId> internalPathId) const override;
+};
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp
new file mode 100644
index 00000000000..ca5d8c22b20
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp
@@ -0,0 +1,58 @@
+#include "source.h"
+
+#include <ydb/core/formats/arrow/accessor/plain/accessor.h>
+#include <ydb/core/tx/columnshard/engines/storage/granule/granule.h>
+#include <ydb/core/tx/conveyor_composite/usage/service.h>
+
+#include <ydb/library/formats/arrow/switch/switch_type.h>
+
+#include <util/system/hostname.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+std::shared_ptr<arrow::Array> TSourceData::BuildArrayAccessor(const ui64 columnId, const ui32 recordsCount) const {
+ // PrimaryIndexSchemaStats
+ if (columnId == 1) {
+ return NArrow::TStatusValidator::GetValid(arrow::MakeArrayFromScalar(arrow::UInt64Scalar(GetTabletId()), recordsCount));
+ }
+ if (columnId == 2) {
+ auto builder = NArrow::MakeBuilder(arrow::uint64());
+ for (auto&& i : Schemas) {
+ NArrow::Append<arrow::UInt64Type>(*builder, i->GetIndexInfo().GetPresetId());
+ }
+ return NArrow::FinishBuilder(std::move(builder));
+ }
+ if (columnId == 3) {
+ auto builder = NArrow::MakeBuilder(arrow::uint64());
+ for (auto&& i : Schemas) {
+ NArrow::Append<arrow::UInt64Type>(*builder, i->GetVersion());
+ }
+ return NArrow::FinishBuilder(std::move(builder));
+ }
+ if (columnId == 4) {
+ auto builder = NArrow::MakeBuilder(arrow::uint64());
+ for (auto&& i : Schemas) {
+ NArrow::Append<arrow::UInt64Type>(*builder, i->GetSnapshot().GetPlanStep());
+ }
+ return NArrow::FinishBuilder(std::move(builder));
+ }
+ if (columnId == 5) {
+ auto builder = NArrow::MakeBuilder(arrow::uint64());
+ for (auto&& i : Schemas) {
+ NArrow::Append<arrow::UInt64Type>(*builder, i->GetSnapshot().GetTxId());
+ }
+ return NArrow::FinishBuilder(std::move(builder));
+ }
+ if (columnId == 6) {
+ auto builder = NArrow::MakeBuilder(arrow::utf8());
+ for (auto&& i : Schemas) {
+ const TString jsonDescription = i->DebugJson().GetStringRobust();
+ NArrow::Append<arrow::StringType>(*builder, arrow::util::string_view(jsonDescription.data(), jsonDescription.size()));
+ }
+ return NArrow::FinishBuilder(std::move(builder));
+ }
+
+ AFL_VERIFY(false)("column_id", columnId);
+ return nullptr;
+}
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h
new file mode 100644
index 00000000000..0c35f42d477
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h
@@ -0,0 +1,31 @@
+#pragma once
+#include <ydb/core/tx/columnshard/engines/reader/common_reader/iterator/constructor.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h>
+#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/source.h>
+
+#include <ydb/library/formats/arrow/simple_arrays_cache.h>
+
+namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas {
+
+class TSourceData: public NAbstract::TTabletSourceData {
+private:
+ using TBase = NAbstract::TTabletSourceData;
+ std::vector<ISnapshotSchema::TPtr> Schemas;
+
+ virtual std::shared_ptr<arrow::Array> BuildArrayAccessor(const ui64 columnId, const ui32 recordsCount) const override;
+
+ virtual void InitUsedRawBytes() override {
+ AFL_VERIFY(!UsedRawBytes);
+ UsedRawBytes = 0;
+ }
+
+public:
+ TSourceData(const ui32 sourceId, const ui32 sourceIdx, const ui64 tabletId, std::vector<ISnapshotSchema::TPtr>&& schemas,
+ NArrow::TSimpleRow&& start, NArrow::TSimpleRow&& finish, const std::shared_ptr<NReader::NSimple::TSpecialReadContext>& context)
+ : TBase(
+ sourceId, sourceIdx, tabletId, std::move(start), std::move(finish), schemas.size(), TSnapshot::Zero(), TSnapshot::Zero(), context)
+ , Schemas(std::move(schemas)) {
+ }
+};
+
+} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make
new file mode 100644
index 00000000000..cc126b1ef77
--- /dev/null
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+SRCS(
+ source.cpp
+ constructor.cpp
+ metadata.cpp
+ GLOBAL schema.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/columnshard/engines/reader/common_reader/iterator
+)
+
+END()
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make
index 41dd8f972e4..43e8bf8a58a 100644
--- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make
@@ -9,6 +9,7 @@ PEERDIR(
ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks
ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules
ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer
+ ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas
)
END()
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h
index c5cf0229c6a..075ecc4eb2c 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h
@@ -25,6 +25,7 @@ class TWritePortionInfoWithBlobsResult;
class ISnapshotSchema {
protected:
virtual TString DoDebugString() const = 0;
+ virtual NJson::TJsonValue DoDebugJson() const = 0;
public:
using TPtr = std::shared_ptr<ISnapshotSchema>;
@@ -72,6 +73,10 @@ public:
std::shared_ptr<arrow::Field> GetFieldByColumnIdOptional(const ui32 columnId) const;
std::shared_ptr<arrow::Field> GetFieldByColumnIdVerified(const ui32 columnId) const;
+ NJson::TJsonValue DebugJson() const {
+ return DoDebugJson();
+ }
+
TString DebugString() const {
return DoDebugString();
}
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp
index a371806b234..8fb1a80f8cc 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp
@@ -83,4 +83,13 @@ TString TFilteredSnapshotSchema::DoDebugString() const {
;
}
+NJson::TJsonValue TFilteredSnapshotSchema::DoDebugJson() const {
+ NJson::TJsonValue result = NJson::JSON_MAP;
+ result.InsertValue("schema", Schema->ToString());
+ result.InsertValue("snapshot", GetSnapshot().DebugString());
+ result.InsertValue("column_ids", JoinSeq(",", ColumnIds));
+ result.InsertValue("index_info", GetIndexInfo().DebugString());
+ return result;
+}
+
}
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h
index 417fcaa0775..1996222e351 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h
@@ -14,6 +14,9 @@ class TFilteredSnapshotSchema: public ISnapshotSchema {
protected:
virtual TString DoDebugString() const override;
+
+ virtual NJson::TJsonValue DoDebugJson() const override;
+
public:
TFilteredSnapshotSchema(const ISnapshotSchema::TPtr& originalSnapshot, const std::vector<ui32>& columnIds);
TFilteredSnapshotSchema(const ISnapshotSchema::TPtr& originalSnapshot, const std::set<ui32>& columnIds);
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h b/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h
index 55d44f17a1a..7c4dd4ab2ba 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h
@@ -15,6 +15,10 @@ private:
std::shared_ptr<TSchemaObjectsCache> SchemaObjectsCache;
public:
+ const THashMap<ui64, std::shared_ptr<TVersionedIndex>> GetPresetVersionedIndex() const {
+ return PresetVersionedIndex;
+ }
+
void RegisterPreset(const ui64 presetId) {
AFL_VERIFY(PresetVersionedIndex.emplace(presetId, std::make_shared<TVersionedIndex>()).second)("preset_id", presetId);
}
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h
index 0cf6aa147d6..bc1dcc4c583 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h
@@ -16,12 +16,19 @@ private:
protected:
virtual TString DoDebugString() const override {
return TStringBuilder() << "("
- "schema=" << Schema->ToString() << ";" <<
- "snapshot=" << Snapshot.DebugString() << ";" <<
- "index_info=" << IndexInfo->DebugString() << ";" <<
- ")"
- ;
+ "schema="
+ << Schema->ToString() << ";" << "snapshot=" << Snapshot.DebugString() << ";"
+ << "index_info=" << IndexInfo->DebugString() << ";" << ")";
}
+
+ virtual NJson::TJsonValue DoDebugJson() const override {
+ NJson::TJsonValue result = NJson::JSON_MAP;
+ result.InsertValue("schema", Schema->ToString());
+ result.InsertValue("snapshot", Snapshot.DebugString());
+ result.InsertValue("index_info", IndexInfo->DebugString());
+ return result;
+ }
+
public:
TSnapshotSchema(TObjectCache<TSchemaVersionId, TIndexInfo>::TEntryGuard&& indexInfo, const TSnapshot& snapshot);
diff --git a/ydb/services/ydb/ydb_logstore_ut.cpp b/ydb/services/ydb/ydb_logstore_ut.cpp
index 1cc17572f37..5c43979cfb3 100644
--- a/ydb/services/ydb/ydb_logstore_ut.cpp
+++ b/ydb/services/ydb/ydb_logstore_ut.cpp
@@ -335,11 +335,12 @@ Y_UNIT_TEST_SUITE(YdbLogStore) {
auto res = schemaClient.ListDirectory("/Root/LogStore/.sys").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString());
auto children = res.GetChildren();
- UNIT_ASSERT_VALUES_EQUAL(children.size(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(children.size(), 5);
UNIT_ASSERT_VALUES_EQUAL(children[0].Name, "store_primary_index_granule_stats");
UNIT_ASSERT_VALUES_EQUAL(children[1].Name, "store_primary_index_optimizer_stats");
UNIT_ASSERT_VALUES_EQUAL(children[2].Name, "store_primary_index_portion_stats");
- UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "store_primary_index_stats");
+ UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "store_primary_index_schema_stats");
+ UNIT_ASSERT_VALUES_EQUAL(children[4].Name, "store_primary_index_stats");
}
{
@@ -356,11 +357,12 @@ Y_UNIT_TEST_SUITE(YdbLogStore) {
auto res = schemaClient.ListDirectory("/Root/LogStore/log1/.sys").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString());
auto children = res.GetChildren();
- UNIT_ASSERT_VALUES_EQUAL(children.size(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(children.size(), 5);
UNIT_ASSERT_VALUES_EQUAL(children[0].Name, "primary_index_granule_stats");
UNIT_ASSERT_VALUES_EQUAL(children[1].Name, "primary_index_optimizer_stats");
UNIT_ASSERT_VALUES_EQUAL(children[2].Name, "primary_index_portion_stats");
- UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "primary_index_stats");
+ UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "primary_index_schema_stats");
+ UNIT_ASSERT_VALUES_EQUAL(children[4].Name, "primary_index_stats");
}
{