diff options
author | ivanmorozov333 <[email protected]> | 2025-07-10 14:59:12 +0300 |
---|---|---|
committer | GitHub <[email protected]> | 2025-07-10 14:59:12 +0300 |
commit | 666035fb9411bf8f58bfdba3a24cf82fed7eed37 (patch) | |
tree | 91f8a735a26c5d6416a4d55b51083ec82b3a3106 | |
parent | 6e61d6058a51532b7fae1fa03aa3e6479e3dc5ba (diff) |
schemas sysview (#20917)
19 files changed, 477 insertions, 9 deletions
diff --git a/ydb/core/kqp/ut/olap/write_ut.cpp b/ydb/core/kqp/ut/olap/write_ut.cpp index 60739b07689..5d3f16282cd 100644 --- a/ydb/core/kqp/ut/olap/write_ut.cpp +++ b/ydb/core/kqp/ut/olap/write_ut.cpp @@ -185,6 +185,9 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { READ: SELECT Col1, Col4, Col5, Col7, Col8 FROM `/Root/ColumnTable` ORDER BY Col1; EXPECTED: [[1u;#;#;#;#];[2u;#;#;#;#];[3u;#;#;#;#];[4u;[4u];#;#;#];[5u;[5u];[5u];#;#];[6u;[6u];[6u];#;#];[7u;[7u];[7u];[7u];#];[8u;[8u];[8u];[8u];[8u]];[9u;[9u];[9u];[9u];[9u]]] ------ + READ: SELECT SchemaVersion FROM `/Root/ColumnTable/.sys/primary_index_schema_stats` WHERE PresetId = 0 ORDER BY SchemaVersion; + EXPECTED: [[[6u]];[[9u]];[[11u]]] + ------ DATA: DELETE FROM `/Root/ColumnTable` ------ diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/schema.cpp index 51cb307bee5..4462c0abec7 100644 --- a/ydb/core/sys_view/common/schema.cpp +++ b/ydb/core/sys_view/common/schema.cpp @@ -319,10 +319,12 @@ private: RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName, ESysViewType::EQueryMetricsOneMinute); RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName); + RegisterOlapStoreSystemView<Schema::PrimaryIndexSchemaStats>(StorePrimaryIndexSchemaStatsName); RegisterOlapStoreSystemView<Schema::PrimaryIndexPortionStats>(StorePrimaryIndexPortionStatsName); RegisterOlapStoreSystemView<Schema::PrimaryIndexGranuleStats>(StorePrimaryIndexGranuleStatsName); RegisterOlapStoreSystemView<Schema::PrimaryIndexOptimizerStats>(StorePrimaryIndexOptimizerStatsName); RegisterColumnTableSystemView<Schema::PrimaryIndexStats>(TablePrimaryIndexStatsName); + RegisterColumnTableSystemView<Schema::PrimaryIndexSchemaStats>(TablePrimaryIndexSchemaStatsName); RegisterColumnTableSystemView<Schema::PrimaryIndexPortionStats>(TablePrimaryIndexPortionStatsName); RegisterColumnTableSystemView<Schema::PrimaryIndexGranuleStats>(TablePrimaryIndexGranuleStatsName); RegisterColumnTableSystemView<Schema::PrimaryIndexOptimizerStats>(TablePrimaryIndexOptimizerStatsName); diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index f26af071cc7..b23d27a6145 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -36,10 +36,12 @@ constexpr TStringBuf TabletsName = "hive_tablets"; constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute"; constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats"; +constexpr TStringBuf StorePrimaryIndexSchemaStatsName = "store_primary_index_schema_stats"; constexpr TStringBuf StorePrimaryIndexPortionStatsName = "store_primary_index_portion_stats"; constexpr TStringBuf StorePrimaryIndexGranuleStatsName = "store_primary_index_granule_stats"; constexpr TStringBuf StorePrimaryIndexOptimizerStatsName = "store_primary_index_optimizer_stats"; constexpr TStringBuf TablePrimaryIndexStatsName = "primary_index_stats"; +constexpr TStringBuf TablePrimaryIndexSchemaStatsName = "primary_index_schema_stats"; constexpr TStringBuf TablePrimaryIndexPortionStatsName = "primary_index_portion_stats"; constexpr TStringBuf TablePrimaryIndexGranuleStatsName = "primary_index_granule_stats"; constexpr TStringBuf TablePrimaryIndexOptimizerStatsName = "primary_index_optimizer_stats"; @@ -804,6 +806,26 @@ struct Schema : NIceDb::Schema { IndexSize, FollowerId>; }; + + struct PrimaryIndexSchemaStats : Table<24> { + struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct PresetId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct SchemaVersion : Column<3, NScheme::NTypeIds::Uint64> {}; + struct SchemaSnapshotPlanStep : Column<4, NScheme::NTypeIds::Uint64> {}; + struct SchemaSnapshotTxId : Column<5, NScheme::NTypeIds::Uint64> {}; + struct SchemaDetails : Column<6, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey<TabletId, PresetId, SchemaVersion>; + using TColumns = TableColumns< + TabletId, + PresetId, + SchemaVersion, + SchemaSnapshotPlanStep, + SchemaSnapshotTxId, + SchemaDetails + >; + }; + }; bool MaybeSystemViewPath(const TVector<TString>& path); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp new file mode 100644 index 00000000000..626b2a405f3 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.cpp @@ -0,0 +1,40 @@ +#include "constructor.h" + +#include <ydb/core/tx/columnshard/engines/column_engine_logs.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { + +TConstructor::TConstructor( + const IColumnEngine& engine, const ui64 tabletId, const std::shared_ptr<NOlap::TPKRangesFilter>& pkFilter, const bool isReverseSort) + : TabletId(tabletId) { + const TColumnEngineForLogs* engineImpl = dynamic_cast<const TColumnEngineForLogs*>(&engine); + std::vector<ISnapshotSchema::TPtr> schemasAll; + for (auto&& i : engineImpl->GetVersionedSchemas().GetPresetVersionedIndex()) { + for (auto&& [_, s] : i.second->GetSnapshotByVersions()) { + schemasAll.emplace_back(s); + } + } + const auto pred = [](const ISnapshotSchema::TPtr& l, const ISnapshotSchema::TPtr& r) { + return std::tuple(l->GetIndexInfo().GetPresetId(), l->GetVersion()) < std::tuple(r->GetIndexInfo().GetPresetId(), r->GetVersion()); + }; + std::sort(schemasAll.begin(), schemasAll.end(), pred); + std::vector<ISnapshotSchema::TPtr> current; + for (auto&& i : schemasAll) { + current.emplace_back(i); + if (current.size() == 10) { + AddConstructors(std::move(current), pkFilter); + current.clear(); + } + } + if (current.size()) { + AddConstructors(std::move(current), pkFilter); + current.clear(); + } + + std::sort(Constructors.begin(), Constructors.end(), TDataConstructor::TComparator(isReverseSort)); + for (ui32 idx = 0; idx < Constructors.size(); ++idx) { + Constructors[idx].SetIndex(idx); + } +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h new file mode 100644 index 00000000000..2055085a8c8 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/constructor.h @@ -0,0 +1,107 @@ +#pragma once +#include "schema.h" +#include "source.h" + +#include <ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/constructor.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { +class TDataConstructor { +private: + ui64 TabletId; + std::vector<ISnapshotSchema::TPtr> Schemas; + NArrow::TSimpleRow Start; + NArrow::TSimpleRow Finish; + ui32 SourceId = 0; + ui32 SourceIdx = 0; + +public: + void SetIndex(const ui32 index) { + AFL_VERIFY(!SourceId); + SourceIdx = index; + SourceId = index + 1; + } + + TDataConstructor(const ui64 tabletId, std::vector<ISnapshotSchema::TPtr>&& schemas) + : TabletId(tabletId) + , Schemas(std::move(schemas)) + , Start(TSchemaAdapter::GetPKSimpleRow(TabletId, Schemas.front()->GetIndexInfo().GetPresetId(), Schemas.front()->GetVersion())) + , Finish(TSchemaAdapter::GetPKSimpleRow(TabletId, Schemas.back()->GetIndexInfo().GetPresetId(), Schemas.back()->GetVersion())) { + if (Schemas.size() > 1) { + AFL_VERIFY(Start < Finish)("start", Start.DebugString())("finish", Finish.DebugString()); + } + } + + const NArrow::TSimpleRow& GetStart() const { + return Start; + } + const NArrow::TSimpleRow& GetFinish() const { + return Finish; + } + + struct TComparator { + private: + const bool IsReverse; + + public: + TComparator(const bool isReverse) + : IsReverse(isReverse) { + } + + bool operator()(const TDataConstructor& l, const TDataConstructor& r) const { + if (IsReverse) { + return r.Finish < l.Finish; + } else { + return l.Start < r.Start; + } + } + }; + + std::shared_ptr<NReader::NSimple::IDataSource> Construct(const std::shared_ptr<NReader::NSimple::TSpecialReadContext>& context) { + AFL_VERIFY(SourceId); + return std::make_shared<TSourceData>(SourceId, SourceIdx, TabletId, std::move(Schemas), std::move(Start), std::move(Finish), context); + } +}; + +class TConstructor: public NAbstract::ISourcesConstructor { +private: + const ui64 TabletId; + std::deque<TDataConstructor> Constructors; + + virtual void DoClear() override { + Constructors.clear(); + } + virtual void DoAbort() override { + Constructors.clear(); + } + virtual bool DoIsFinished() const override { + return Constructors.empty(); + } + virtual std::shared_ptr<NReader::NCommon::IDataSource> DoExtractNext( + const std::shared_ptr<NReader::NCommon::TSpecialReadContext>& context) override { + AFL_VERIFY(Constructors.size()); + std::shared_ptr<NReader::NCommon::IDataSource> result = + Constructors.front().Construct(std::static_pointer_cast<NReader::NSimple::TSpecialReadContext>(context)); + Constructors.pop_front(); + return result; + } + virtual void DoInitCursor(const std::shared_ptr<IScanCursor>& /*cursor*/) override { + } + virtual TString DoDebugString() const override { + return Default<TString>(); + } + + void AddConstructors(std::vector<ISnapshotSchema::TPtr>&& schemas, const std::shared_ptr<NOlap::TPKRangesFilter>& pkFilter) { + Constructors.emplace_back(TabletId, std::move(schemas)); + if (!pkFilter->IsUsed(Constructors.back().GetStart(), Constructors.back().GetFinish())) { + Constructors.pop_back(); + } + } + +public: + TConstructor( + const IColumnEngine& engine, const ui64 tabletId, const std::shared_ptr<NOlap::TPKRangesFilter>& pkFilter, const bool isReverseSort); +}; +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp new file mode 100644 index 00000000000..6a1386e2c20 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.cpp @@ -0,0 +1,32 @@ +#include "constructor.h" +#include "metadata.h" +#include "schema.h" + +#include <ydb/core/tx/columnshard/engines/reader/common/description.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { + +TAccessor::TAccessor(const TString& tablePath, const NColumnShard::TSchemeShardLocalPathId externalPathId, + const std::optional<NColumnShard::TInternalPathId> internalPathId) + : TBase(tablePath) + , PathId(NColumnShard::TUnifiedPathId::BuildNoCheck(internalPathId, externalPathId)) { + AFL_VERIFY(CheckTablePath(GetTablePath())); +} + +std::unique_ptr<NReader::NCommon::ISourcesConstructor> TAccessor::SelectMetadata(const TSelectMetadataContext& context, + const NReader::TReadDescription& readDescription, const bool /*withUncommitted*/, const bool isPlain) const { + AFL_VERIFY(!isPlain); + return std::make_unique<TConstructor>( + context.GetEngine(), readDescription.GetTabletId(), readDescription.PKRangesFilter, readDescription.IsReverseSort()); +} + +std::shared_ptr<ISnapshotSchema> TAccessor::GetSnapshotSchemaOptional( + const TVersionedPresetSchemas& vSchemas, const TSnapshot& /*snapshot*/) const { + return vSchemas.GetVersionedIndex(TSchemaAdapter::GetInstance().GetPresetId()).GetLastSchema(); +} + +std::shared_ptr<const TVersionedIndex> TAccessor::GetVersionedIndexCopyOptional(TVersionedPresetSchemas& vSchemas) const { + return vSchemas.GetVersionedIndexCopy(TSchemaAdapter::GetInstance().GetPresetId()); +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h new file mode 100644 index 00000000000..4a83ed44ec4 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/metadata.h @@ -0,0 +1,36 @@ +#pragma once +#include <ydb/core/tx/columnshard/engines/metadata_accessor.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/metadata.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { + +class TAccessor: public NAbstract::TAccessor { +private: + using TBase = NAbstract::TAccessor; + const NColumnShard::TUnifiedPathId PathId; + + virtual std::optional<NColumnShard::TUnifiedPathId> GetPathId() const override { + return PathId; + } + + virtual std::shared_ptr<const TVersionedIndex> GetVersionedIndexCopyOptional(TVersionedPresetSchemas& vSchemas) const override; + + virtual std::shared_ptr<ISnapshotSchema> GetSnapshotSchemaOptional( + const TVersionedPresetSchemas& vSchemas, const TSnapshot& snapshot) const override; + +public: + static bool CheckTablePath(const TString& tablePath) { + return tablePath.EndsWith("/.sys/primary_index_schema_stats") || tablePath.EndsWith("/.sys/store_primary_index_schema_stats"); + } + + TAccessor(const TString& tableName, const NColumnShard::TSchemeShardLocalPathId externalPathId, + const std::optional<NColumnShard::TInternalPathId> internalPathId); + virtual std::unique_ptr<NReader::NCommon::ISourcesConstructor> SelectMetadata(const TSelectMetadataContext& context, + const NReader::TReadDescription& readDescription, const bool withUncommitted, const bool isPlain) const override; + virtual std::optional<TGranuleShardingInfo> GetShardingInfo( + const std::shared_ptr<const TVersionedIndex>& /*indexVersionsPointer*/, const NOlap::TSnapshot& /*ss*/) const override { + return std::nullopt; + } +}; + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp new file mode 100644 index 00000000000..187664d52f2 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp @@ -0,0 +1,59 @@ +#include "metadata.h" +#include "schema.h" + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { + +NArrow::TSimpleRow TSchemaAdapter::GetPKSimpleRow(const ui64 tabletId, const ui64 presetId, const ui64 schemaVersion) { + NArrow::TSimpleRowViewV0::TWriter writer(sizeof(ui64) * 2); + writer.Append<ui64>(tabletId); + writer.Append<ui64>(presetId); + writer.Append<ui64>(schemaVersion); + return NArrow::TSimpleRow(writer.Finish(), GetPKSchema()); +} + +std::shared_ptr<arrow::Schema> TSchemaAdapter::GetPKSchema() { + static std::shared_ptr<arrow::Schema> schema = []() { + arrow::FieldVector fields = { std::make_shared<arrow::Field>("TabletId", arrow::uint64()), + std::make_shared<arrow::Field>("PresetId", arrow::uint64()), std::make_shared<arrow::Field>("SchemaVersion", arrow::uint64()) }; + return std::make_shared<arrow::Schema>(std::move(fields)); + }(); + return schema; +} + +TIndexInfo TSchemaAdapter::GetIndexInfo( + const std::shared_ptr<IStoragesManager>& storagesManager, const std::shared_ptr<TSchemaObjectsCache>& schemaObjectsCache) const { + //PrimaryIndexSchemaStats + + static NKikimrSchemeOp::TColumnTableSchema proto = []() { + NKikimrSchemeOp::TColumnTableSchema proto; + ui32 currentId = 0; + const auto pred = [&](const TString& name, const NScheme::TTypeId typeId, const std::optional<ui32> entityId = std::nullopt) { + auto* col = proto.AddColumns(); + col->SetId(entityId.value_or(++currentId)); + col->SetName(name); + col->SetTypeId(typeId); + }; + pred("TabletId", NScheme::NTypeIds::Uint64); + pred("PresetId", NScheme::NTypeIds::Uint64); + pred("SchemaVersion", NScheme::NTypeIds::Uint64); + pred("SchemaSnapshotPlanStep", NScheme::NTypeIds::Uint64); + pred("SchemaSnapshotTxId", NScheme::NTypeIds::Uint64); + pred("SchemaDetails", NScheme::NTypeIds::Utf8); + + proto.AddKeyColumnNames("TabletId"); + proto.AddKeyColumnNames("PresetId"); + proto.AddKeyColumnNames("SchemaVersion"); + return proto; + }(); + + auto indexInfo = TIndexInfo::BuildFromProto(GetPresetId(), proto, storagesManager, schemaObjectsCache); + AFL_VERIFY(indexInfo); + return std::move(*indexInfo); +} + +std::shared_ptr<ITableMetadataAccessor> TSchemaAdapter::BuildMetadataAccessor(const TString& tableName, + const NColumnShard::TSchemeShardLocalPathId externalPathId, const std::optional<NColumnShard::TInternalPathId> internalPathId) const { + return std::make_shared<TAccessor>(tableName, externalPathId, internalPathId); +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h new file mode 100644 index 00000000000..fd974f53176 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.h @@ -0,0 +1,33 @@ +#pragma once +#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> +#include <ydb/core/tx/columnshard/common/path_id.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/schema.h> +#include <ydb/core/tx/columnshard/engines/scheme/index_info.h> +#include <ydb/core/tx/columnshard/engines/scheme/objects_cache.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { + +class TSchemaAdapter: public NAbstract::ISchemaAdapter { +private: + static const inline auto Registrator1 = TFactory::TRegistrator<TSchemaAdapter>("store_primary_index_schema_stats"); + static const inline auto Registrator2 = TFactory::TRegistrator<TSchemaAdapter>("primary_index_schema_stats"); + +public: + static const TSchemaAdapter& GetInstance() { + return *Singleton<TSchemaAdapter>(); + } + + virtual ui64 GetPresetId() const override { + static ui64 presetId = NAbstract::ISchemaAdapter::Counter.Inc(); + return Max<ui64>() - presetId; + } + static NArrow::TSimpleRow GetPKSimpleRow(const ui64 tabletId, const ui64 presetId, const ui64 schemaVersion); + static std::shared_ptr<arrow::Schema> GetPKSchema(); + virtual TIndexInfo GetIndexInfo( + const std::shared_ptr<IStoragesManager>& storagesManager, const std::shared_ptr<TSchemaObjectsCache>& schemaObjectsCache) const override; + virtual std::shared_ptr<ITableMetadataAccessor> BuildMetadataAccessor(const TString& tableName, + const NColumnShard::TSchemeShardLocalPathId externalPathId, + const std::optional<NColumnShard::TInternalPathId> internalPathId) const override; +}; + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp new file mode 100644 index 00000000000..ca5d8c22b20 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp @@ -0,0 +1,58 @@ +#include "source.h" + +#include <ydb/core/formats/arrow/accessor/plain/accessor.h> +#include <ydb/core/tx/columnshard/engines/storage/granule/granule.h> +#include <ydb/core/tx/conveyor_composite/usage/service.h> + +#include <ydb/library/formats/arrow/switch/switch_type.h> + +#include <util/system/hostname.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { +std::shared_ptr<arrow::Array> TSourceData::BuildArrayAccessor(const ui64 columnId, const ui32 recordsCount) const { + // PrimaryIndexSchemaStats + if (columnId == 1) { + return NArrow::TStatusValidator::GetValid(arrow::MakeArrayFromScalar(arrow::UInt64Scalar(GetTabletId()), recordsCount)); + } + if (columnId == 2) { + auto builder = NArrow::MakeBuilder(arrow::uint64()); + for (auto&& i : Schemas) { + NArrow::Append<arrow::UInt64Type>(*builder, i->GetIndexInfo().GetPresetId()); + } + return NArrow::FinishBuilder(std::move(builder)); + } + if (columnId == 3) { + auto builder = NArrow::MakeBuilder(arrow::uint64()); + for (auto&& i : Schemas) { + NArrow::Append<arrow::UInt64Type>(*builder, i->GetVersion()); + } + return NArrow::FinishBuilder(std::move(builder)); + } + if (columnId == 4) { + auto builder = NArrow::MakeBuilder(arrow::uint64()); + for (auto&& i : Schemas) { + NArrow::Append<arrow::UInt64Type>(*builder, i->GetSnapshot().GetPlanStep()); + } + return NArrow::FinishBuilder(std::move(builder)); + } + if (columnId == 5) { + auto builder = NArrow::MakeBuilder(arrow::uint64()); + for (auto&& i : Schemas) { + NArrow::Append<arrow::UInt64Type>(*builder, i->GetSnapshot().GetTxId()); + } + return NArrow::FinishBuilder(std::move(builder)); + } + if (columnId == 6) { + auto builder = NArrow::MakeBuilder(arrow::utf8()); + for (auto&& i : Schemas) { + const TString jsonDescription = i->DebugJson().GetStringRobust(); + NArrow::Append<arrow::StringType>(*builder, arrow::util::string_view(jsonDescription.data(), jsonDescription.size())); + } + return NArrow::FinishBuilder(std::move(builder)); + } + + AFL_VERIFY(false)("column_id", columnId); + return nullptr; +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h new file mode 100644 index 00000000000..0c35f42d477 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.h @@ -0,0 +1,31 @@ +#pragma once +#include <ydb/core/tx/columnshard/engines/reader/common_reader/iterator/constructor.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h> +#include <ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/abstract/source.h> + +#include <ydb/library/formats/arrow/simple_arrays_cache.h> + +namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { + +class TSourceData: public NAbstract::TTabletSourceData { +private: + using TBase = NAbstract::TTabletSourceData; + std::vector<ISnapshotSchema::TPtr> Schemas; + + virtual std::shared_ptr<arrow::Array> BuildArrayAccessor(const ui64 columnId, const ui32 recordsCount) const override; + + virtual void InitUsedRawBytes() override { + AFL_VERIFY(!UsedRawBytes); + UsedRawBytes = 0; + } + +public: + TSourceData(const ui32 sourceId, const ui32 sourceIdx, const ui64 tabletId, std::vector<ISnapshotSchema::TPtr>&& schemas, + NArrow::TSimpleRow&& start, NArrow::TSimpleRow&& finish, const std::shared_ptr<NReader::NSimple::TSpecialReadContext>& context) + : TBase( + sourceId, sourceIdx, tabletId, std::move(start), std::move(finish), schemas.size(), TSnapshot::Zero(), TSnapshot::Zero(), context) + , Schemas(std::move(schemas)) { + } +}; + +} // namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make new file mode 100644 index 00000000000..cc126b1ef77 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + source.cpp + constructor.cpp + metadata.cpp + GLOBAL schema.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/engines/reader/common_reader/iterator +) + +END() diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make index 41dd8f972e4..43e8bf8a58a 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/ya.make @@ -9,6 +9,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer + ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas ) END() diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h index c5cf0229c6a..075ecc4eb2c 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h @@ -25,6 +25,7 @@ class TWritePortionInfoWithBlobsResult; class ISnapshotSchema { protected: virtual TString DoDebugString() const = 0; + virtual NJson::TJsonValue DoDebugJson() const = 0; public: using TPtr = std::shared_ptr<ISnapshotSchema>; @@ -72,6 +73,10 @@ public: std::shared_ptr<arrow::Field> GetFieldByColumnIdOptional(const ui32 columnId) const; std::shared_ptr<arrow::Field> GetFieldByColumnIdVerified(const ui32 columnId) const; + NJson::TJsonValue DebugJson() const { + return DoDebugJson(); + } + TString DebugString() const { return DoDebugString(); } diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp index a371806b234..8fb1a80f8cc 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp @@ -83,4 +83,13 @@ TString TFilteredSnapshotSchema::DoDebugString() const { ; } +NJson::TJsonValue TFilteredSnapshotSchema::DoDebugJson() const { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("schema", Schema->ToString()); + result.InsertValue("snapshot", GetSnapshot().DebugString()); + result.InsertValue("column_ids", JoinSeq(",", ColumnIds)); + result.InsertValue("index_info", GetIndexInfo().DebugString()); + return result; +} + } diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h index 417fcaa0775..1996222e351 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h @@ -14,6 +14,9 @@ class TFilteredSnapshotSchema: public ISnapshotSchema { protected: virtual TString DoDebugString() const override; + + virtual NJson::TJsonValue DoDebugJson() const override; + public: TFilteredSnapshotSchema(const ISnapshotSchema::TPtr& originalSnapshot, const std::vector<ui32>& columnIds); TFilteredSnapshotSchema(const ISnapshotSchema::TPtr& originalSnapshot, const std::set<ui32>& columnIds); diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h b/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h index 55d44f17a1a..7c4dd4ab2ba 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/preset_schemas.h @@ -15,6 +15,10 @@ private: std::shared_ptr<TSchemaObjectsCache> SchemaObjectsCache; public: + const THashMap<ui64, std::shared_ptr<TVersionedIndex>> GetPresetVersionedIndex() const { + return PresetVersionedIndex; + } + void RegisterPreset(const ui64 presetId) { AFL_VERIFY(PresetVersionedIndex.emplace(presetId, std::make_shared<TVersionedIndex>()).second)("preset_id", presetId); } diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h index 0cf6aa147d6..bc1dcc4c583 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h @@ -16,12 +16,19 @@ private: protected: virtual TString DoDebugString() const override { return TStringBuilder() << "(" - "schema=" << Schema->ToString() << ";" << - "snapshot=" << Snapshot.DebugString() << ";" << - "index_info=" << IndexInfo->DebugString() << ";" << - ")" - ; + "schema=" + << Schema->ToString() << ";" << "snapshot=" << Snapshot.DebugString() << ";" + << "index_info=" << IndexInfo->DebugString() << ";" << ")"; } + + virtual NJson::TJsonValue DoDebugJson() const override { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("schema", Schema->ToString()); + result.InsertValue("snapshot", Snapshot.DebugString()); + result.InsertValue("index_info", IndexInfo->DebugString()); + return result; + } + public: TSnapshotSchema(TObjectCache<TSchemaVersionId, TIndexInfo>::TEntryGuard&& indexInfo, const TSnapshot& snapshot); diff --git a/ydb/services/ydb/ydb_logstore_ut.cpp b/ydb/services/ydb/ydb_logstore_ut.cpp index 1cc17572f37..5c43979cfb3 100644 --- a/ydb/services/ydb/ydb_logstore_ut.cpp +++ b/ydb/services/ydb/ydb_logstore_ut.cpp @@ -335,11 +335,12 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { auto res = schemaClient.ListDirectory("/Root/LogStore/.sys").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); auto children = res.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 5); UNIT_ASSERT_VALUES_EQUAL(children[0].Name, "store_primary_index_granule_stats"); UNIT_ASSERT_VALUES_EQUAL(children[1].Name, "store_primary_index_optimizer_stats"); UNIT_ASSERT_VALUES_EQUAL(children[2].Name, "store_primary_index_portion_stats"); - UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "store_primary_index_stats"); + UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "store_primary_index_schema_stats"); + UNIT_ASSERT_VALUES_EQUAL(children[4].Name, "store_primary_index_stats"); } { @@ -356,11 +357,12 @@ Y_UNIT_TEST_SUITE(YdbLogStore) { auto res = schemaClient.ListDirectory("/Root/LogStore/log1/.sys").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); auto children = res.GetChildren(); - UNIT_ASSERT_VALUES_EQUAL(children.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(children.size(), 5); UNIT_ASSERT_VALUES_EQUAL(children[0].Name, "primary_index_granule_stats"); UNIT_ASSERT_VALUES_EQUAL(children[1].Name, "primary_index_optimizer_stats"); UNIT_ASSERT_VALUES_EQUAL(children[2].Name, "primary_index_portion_stats"); - UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "primary_index_stats"); + UNIT_ASSERT_VALUES_EQUAL(children[3].Name, "primary_index_schema_stats"); + UNIT_ASSERT_VALUES_EQUAL(children[4].Name, "primary_index_stats"); } { |