aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov <ivanmorozov@yandex-team.com>2023-06-09 16:03:18 +0300
committerivanmorozov <ivanmorozov@yandex-team.com>2023-06-09 16:03:18 +0300
commitb2f1df9368a06c27dfed79ecd6c048e4dc6856a0 (patch)
treedb842617d4271700272a524ae1120f3ad47289b7
parent4e48f42bb1da68482885363f334d1f317a51f78c (diff)
downloadydb-b2f1df9368a06c27dfed79ecd6c048e4dc6856a0.tar.gz
signals and composite on + tests
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.h1
-rw-r--r--ydb/core/kqp/ut/olap/kqp_olap_ut.cpp11
-rw-r--r--ydb/core/testlib/CMakeLists.darwin-x86_64.txt2
-rw-r--r--ydb/core/testlib/CMakeLists.linux-aarch64.txt2
-rw-r--r--ydb/core/testlib/CMakeLists.linux-x86_64.txt2
-rw-r--r--ydb/core/testlib/CMakeLists.windows-x86_64.txt2
-rw-r--r--ydb/core/testlib/controllers/CMakeLists.darwin-x86_64.txt17
-rw-r--r--ydb/core/testlib/controllers/CMakeLists.linux-aarch64.txt18
-rw-r--r--ydb/core/testlib/controllers/CMakeLists.linux-x86_64.txt18
-rw-r--r--ydb/core/testlib/controllers/CMakeLists.txt17
-rw-r--r--ydb/core/testlib/controllers/CMakeLists.windows-x86_64.txt17
-rw-r--r--ydb/core/testlib/controllers/abstract.cpp4
-rw-r--r--ydb/core/testlib/controllers/abstract.h44
-rw-r--r--ydb/core/tx/columnshard/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/tx/columnshard/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/counters/engine_logs.h9
-rw-r--r--ydb/core/tx/columnshard/engines/insert_table/path_info.cpp2
-rw-r--r--ydb/core/tx/columnshard/engines/reader/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/reader/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/reader/read_filter_merger.cpp1
-rw-r--r--ydb/core/tx/columnshard/engines/reader/read_metadata.cpp15
-rw-r--r--ydb/core/tx/columnshard/engines/reader/read_metadata.h1
-rw-r--r--ydb/core/tx/columnshard/engines/storage/granule.cpp2
-rw-r--r--ydb/core/tx/columnshard/testlib/CMakeLists.darwin-x86_64.txt19
-rw-r--r--ydb/core/tx/columnshard/testlib/CMakeLists.linux-aarch64.txt20
-rw-r--r--ydb/core/tx/columnshard/testlib/CMakeLists.linux-x86_64.txt20
-rw-r--r--ydb/core/tx/columnshard/testlib/CMakeLists.txt17
-rw-r--r--ydb/core/tx/columnshard/testlib/CMakeLists.windows-x86_64.txt19
-rw-r--r--ydb/core/tx/columnshard/testlib/controller.cpp18
-rw-r--r--ydb/core/tx/columnshard/testlib/controller.h18
34 files changed, 310 insertions, 14 deletions
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h
index 30dc781d1c0..f5c270c921a 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.h
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.h
@@ -85,6 +85,7 @@ struct TKikimrSettings: public TTestFeatureFlagsHolder<TKikimrSettings> {
TKikimrSettings()
{
+ FeatureFlags.SetForceColumnTablesCompositeMarks(true);
}
TKikimrSettings& SetAppConfig(const NKikimrConfig::TAppConfig& value) { AppConfig = value; return *this; }
diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
index ce224790a20..46efc1ccb74 100644
--- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
+++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
@@ -13,6 +13,8 @@
#include <ydb/core/formats/arrow/simple_builder/batch.h>
#include <ydb/core/formats/arrow/ssa_runtime_version.h>
#include <ydb/core/kqp/executer_actor/kqp_executer.h>
+#include <ydb/core/testlib/controllers/abstract.h>
+#include <ydb/core/tx/columnshard/testlib/controller.h>
#include <ydb/core/tx/datashard/datashard.h>
#include <ydb/core/tx/datashard/datashard_ut_common_kqp.h>
#include <ydb/core/tx/datashard/datashard_ut_common.h>
@@ -1349,10 +1351,14 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
(`timestamp` < CAST(1000100 AS Timestamp) AND `timestamp` > CAST(1000095 AS Timestamp)) OR
(`timestamp` <= CAST(1001000 AS Timestamp) AND `timestamp` >= CAST(1000999 AS Timestamp)) OR
(`timestamp` > CAST(1002000 AS Timestamp))
- ORDER BY `timestamp`;
+ ORDER BY `timestamp`
+ LIMIT 1000;
)");
+ auto csController = NYDBTest::TControllers::RegisterCSController<NYDBTest::NColumnShard::TController>();
auto rows = ExecuteScanQuery(tableClient, selectQuery);
+ UNIT_ASSERT(csController->HasPKSortingOnly());
+
TInstant tsPrev = TInstant::MicroSeconds(1000000);
std::set<ui64> results = { 1000096, 1000097, 1000098, 1000099, 1000999, 1001000 };
@@ -1393,7 +1399,10 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
LIMIT 1000;
)");
+ auto csController = NYDBTest::TControllers::RegisterCSController<NYDBTest::NColumnShard::TController>();
auto rows = ExecuteScanQuery(tableClient, selectQuery);
+ UNIT_ASSERT(csController->HasPKSortingOnly());
+
TInstant tsPrev = TInstant::MicroSeconds(2000000);
std::set<ui64> results = { 1000096, 1000097, 1000098, 1000099,
1000999, 1001000,
diff --git a/ydb/core/testlib/CMakeLists.darwin-x86_64.txt b/ydb/core/testlib/CMakeLists.darwin-x86_64.txt
index 485e4984e46..3587ce18424 100644
--- a/ydb/core/testlib/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/testlib/CMakeLists.darwin-x86_64.txt
@@ -8,6 +8,7 @@
add_subdirectory(actors)
add_subdirectory(basics)
+add_subdirectory(controllers)
add_subdirectory(default)
add_subdirectory(pg)
@@ -59,6 +60,7 @@ target_link_libraries(ydb-core-testlib PUBLIC
core-sys_view-processor
core-sys_view-service
core-testlib-actors
+ core-testlib-controllers
core-testlib-basics
core-tx-columnshard
core-tx-coordinator
diff --git a/ydb/core/testlib/CMakeLists.linux-aarch64.txt b/ydb/core/testlib/CMakeLists.linux-aarch64.txt
index 1d8990c7e80..10ccf60904c 100644
--- a/ydb/core/testlib/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/testlib/CMakeLists.linux-aarch64.txt
@@ -8,6 +8,7 @@
add_subdirectory(actors)
add_subdirectory(basics)
+add_subdirectory(controllers)
add_subdirectory(default)
add_subdirectory(pg)
@@ -60,6 +61,7 @@ target_link_libraries(ydb-core-testlib PUBLIC
core-sys_view-processor
core-sys_view-service
core-testlib-actors
+ core-testlib-controllers
core-testlib-basics
core-tx-columnshard
core-tx-coordinator
diff --git a/ydb/core/testlib/CMakeLists.linux-x86_64.txt b/ydb/core/testlib/CMakeLists.linux-x86_64.txt
index 1d8990c7e80..10ccf60904c 100644
--- a/ydb/core/testlib/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/testlib/CMakeLists.linux-x86_64.txt
@@ -8,6 +8,7 @@
add_subdirectory(actors)
add_subdirectory(basics)
+add_subdirectory(controllers)
add_subdirectory(default)
add_subdirectory(pg)
@@ -60,6 +61,7 @@ target_link_libraries(ydb-core-testlib PUBLIC
core-sys_view-processor
core-sys_view-service
core-testlib-actors
+ core-testlib-controllers
core-testlib-basics
core-tx-columnshard
core-tx-coordinator
diff --git a/ydb/core/testlib/CMakeLists.windows-x86_64.txt b/ydb/core/testlib/CMakeLists.windows-x86_64.txt
index 485e4984e46..3587ce18424 100644
--- a/ydb/core/testlib/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/testlib/CMakeLists.windows-x86_64.txt
@@ -8,6 +8,7 @@
add_subdirectory(actors)
add_subdirectory(basics)
+add_subdirectory(controllers)
add_subdirectory(default)
add_subdirectory(pg)
@@ -59,6 +60,7 @@ target_link_libraries(ydb-core-testlib PUBLIC
core-sys_view-processor
core-sys_view-service
core-testlib-actors
+ core-testlib-controllers
core-testlib-basics
core-tx-columnshard
core-tx-coordinator
diff --git a/ydb/core/testlib/controllers/CMakeLists.darwin-x86_64.txt b/ydb/core/testlib/controllers/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..ee443af37ac
--- /dev/null
+++ b/ydb/core/testlib/controllers/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-testlib-controllers)
+target_link_libraries(core-testlib-controllers PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(core-testlib-controllers PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/testlib/controllers/abstract.cpp
+)
diff --git a/ydb/core/testlib/controllers/CMakeLists.linux-aarch64.txt b/ydb/core/testlib/controllers/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..16031042250
--- /dev/null
+++ b/ydb/core/testlib/controllers/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-testlib-controllers)
+target_link_libraries(core-testlib-controllers PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(core-testlib-controllers PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/testlib/controllers/abstract.cpp
+)
diff --git a/ydb/core/testlib/controllers/CMakeLists.linux-x86_64.txt b/ydb/core/testlib/controllers/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..16031042250
--- /dev/null
+++ b/ydb/core/testlib/controllers/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-testlib-controllers)
+target_link_libraries(core-testlib-controllers PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(core-testlib-controllers PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/testlib/controllers/abstract.cpp
+)
diff --git a/ydb/core/testlib/controllers/CMakeLists.txt b/ydb/core/testlib/controllers/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/core/testlib/controllers/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/core/testlib/controllers/CMakeLists.windows-x86_64.txt b/ydb/core/testlib/controllers/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..ee443af37ac
--- /dev/null
+++ b/ydb/core/testlib/controllers/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-testlib-controllers)
+target_link_libraries(core-testlib-controllers PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(core-testlib-controllers PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/testlib/controllers/abstract.cpp
+)
diff --git a/ydb/core/testlib/controllers/abstract.cpp b/ydb/core/testlib/controllers/abstract.cpp
new file mode 100644
index 00000000000..a4d916545ea
--- /dev/null
+++ b/ydb/core/testlib/controllers/abstract.cpp
@@ -0,0 +1,4 @@
+#include "abstract.h"
+
+namespace NKikimr::NYDBTest {
+}
diff --git a/ydb/core/testlib/controllers/abstract.h b/ydb/core/testlib/controllers/abstract.h
new file mode 100644
index 00000000000..523dc7451f6
--- /dev/null
+++ b/ydb/core/testlib/controllers/abstract.h
@@ -0,0 +1,44 @@
+#pragma once
+#include <ydb/library/accessor/accessor.h>
+#include <util/generic/singleton.h>
+#include <memory>
+
+namespace NKikimr::NOlap::NIndexedReader {
+class IOrderPolicy;
+}
+
+namespace NKikimr::NYDBTest {
+
+class ICSController {
+private:
+ YDB_READONLY(ui32, OnSortingPolicyCounter, 0);
+protected:
+ virtual bool DoOnSortingPolicy(std::shared_ptr<NOlap::NIndexedReader::IOrderPolicy> /*policy*/) {
+ return true;
+ }
+public:
+ using TPtr = std::shared_ptr<ICSController>;
+ virtual ~ICSController() = default;
+ bool OnSortingPolicy(std::shared_ptr<NOlap::NIndexedReader::IOrderPolicy> policy) {
+ ++OnSortingPolicyCounter;
+ return DoOnSortingPolicy(policy);
+ }
+};
+
+class TControllers {
+private:
+ ICSController::TPtr CSController = std::make_shared<ICSController>();
+public:
+ template <class T, class... Types>
+ static std::shared_ptr<T> RegisterCSController(Types... args) {
+ auto result = std::make_shared<T>(args...);
+ Singleton<TControllers>()->CSController = result;
+ return result;
+ }
+
+ static ICSController::TPtr GetColumnShardController() {
+ return Singleton<TControllers>()->CSController;
+ }
+};
+
+}
diff --git a/ydb/core/tx/columnshard/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/CMakeLists.darwin-x86_64.txt
index 4cc3c30463a..7914110dfd6 100644
--- a/ydb/core/tx/columnshard/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/tx/columnshard/CMakeLists.darwin-x86_64.txt
@@ -8,6 +8,7 @@
add_subdirectory(counters)
add_subdirectory(engines)
+add_subdirectory(testlib)
add_subdirectory(ut_rw)
add_subdirectory(ut_schema)
get_built_tool_path(
diff --git a/ydb/core/tx/columnshard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/CMakeLists.linux-aarch64.txt
index e044cc71d3e..99b3dea1110 100644
--- a/ydb/core/tx/columnshard/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/columnshard/CMakeLists.linux-aarch64.txt
@@ -8,6 +8,7 @@
add_subdirectory(counters)
add_subdirectory(engines)
+add_subdirectory(testlib)
add_subdirectory(ut_rw)
add_subdirectory(ut_schema)
get_built_tool_path(
diff --git a/ydb/core/tx/columnshard/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/CMakeLists.linux-x86_64.txt
index e044cc71d3e..99b3dea1110 100644
--- a/ydb/core/tx/columnshard/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/tx/columnshard/CMakeLists.linux-x86_64.txt
@@ -8,6 +8,7 @@
add_subdirectory(counters)
add_subdirectory(engines)
+add_subdirectory(testlib)
add_subdirectory(ut_rw)
add_subdirectory(ut_schema)
get_built_tool_path(
diff --git a/ydb/core/tx/columnshard/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/CMakeLists.windows-x86_64.txt
index 4cc3c30463a..7914110dfd6 100644
--- a/ydb/core/tx/columnshard/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/tx/columnshard/CMakeLists.windows-x86_64.txt
@@ -8,6 +8,7 @@
add_subdirectory(counters)
add_subdirectory(engines)
+add_subdirectory(testlib)
add_subdirectory(ut_rw)
add_subdirectory(ut_schema)
get_built_tool_path(
diff --git a/ydb/core/tx/columnshard/counters/engine_logs.h b/ydb/core/tx/columnshard/counters/engine_logs.h
index cb33f37fb48..5c64942bc1d 100644
--- a/ydb/core/tx/columnshard/counters/engine_logs.h
+++ b/ydb/core/tx/columnshard/counters/engine_logs.h
@@ -41,11 +41,6 @@ public:
PortionsSize->Set(dataInfo.GetPortionsSize());
PortionsCount->Set(dataInfo.GetPortionsCount());
}
-
- void OnPortionsInfo(const ui64 size, const ui32 chunks) const {
- PortionsSize->Set(size);
- PortionsCount->Set(chunks);
- }
};
class TAgentDataClassCounters: public TCommonCountersOwner {
@@ -79,8 +74,8 @@ public:
{
}
- void OnFullData(const ui64 size, const ui32 chunksCount) const {
- FullData.OnPortionsInfo(size, chunksCount);
+ void OnFullData(const TBaseGranuleDataClassSummary& dataInfo) const {
+ FullData.OnPortionsInfo(dataInfo);
}
void OnInsertedData(const TBaseGranuleDataClassSummary& dataInfo) const {
diff --git a/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp b/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp
index 489ec93cd5f..a68a84a2b13 100644
--- a/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp
+++ b/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp
@@ -29,7 +29,7 @@ void TPathInfo::AddInsertedSize(const i64 size, const ui64 overloadLimit) {
Y_VERIFY(InsertedSize >= 0);
Summary->InsertedSize += size;
Y_VERIFY(Summary->InsertedSize >= 0);
- PathIdCounters.Committed.OnPathIdDataInfo(InsertedSize, 0);
+ PathIdCounters.Inserted.OnPathIdDataInfo(InsertedSize, 0);
SetInsertedOverload((ui64)InsertedSize > overloadLimit);
}
diff --git a/ydb/core/tx/columnshard/engines/reader/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/engines/reader/CMakeLists.darwin-x86_64.txt
index 19f1228ad62..f387bcf9411 100644
--- a/ydb/core/tx/columnshard/engines/reader/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/tx/columnshard/engines/reader/CMakeLists.darwin-x86_64.txt
@@ -25,6 +25,7 @@ target_link_libraries(columnshard-engines-reader PUBLIC
ydb-core-protos
core-formats-arrow
columnshard-engines-predicate
+ tx-columnshard-testlib
core-tx-program
engines-reader-order_control
tools-enum_parser-enum_serialization_runtime
diff --git a/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-aarch64.txt
index 78e360e108d..ce3782e72b7 100644
--- a/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-aarch64.txt
@@ -26,6 +26,7 @@ target_link_libraries(columnshard-engines-reader PUBLIC
ydb-core-protos
core-formats-arrow
columnshard-engines-predicate
+ tx-columnshard-testlib
core-tx-program
engines-reader-order_control
tools-enum_parser-enum_serialization_runtime
diff --git a/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-x86_64.txt
index 78e360e108d..ce3782e72b7 100644
--- a/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/tx/columnshard/engines/reader/CMakeLists.linux-x86_64.txt
@@ -26,6 +26,7 @@ target_link_libraries(columnshard-engines-reader PUBLIC
ydb-core-protos
core-formats-arrow
columnshard-engines-predicate
+ tx-columnshard-testlib
core-tx-program
engines-reader-order_control
tools-enum_parser-enum_serialization_runtime
diff --git a/ydb/core/tx/columnshard/engines/reader/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/engines/reader/CMakeLists.windows-x86_64.txt
index 19f1228ad62..f387bcf9411 100644
--- a/ydb/core/tx/columnshard/engines/reader/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/tx/columnshard/engines/reader/CMakeLists.windows-x86_64.txt
@@ -25,6 +25,7 @@ target_link_libraries(columnshard-engines-reader PUBLIC
ydb-core-protos
core-formats-arrow
columnshard-engines-predicate
+ tx-columnshard-testlib
core-tx-program
engines-reader-order_control
tools-enum_parser-enum_serialization_runtime
diff --git a/ydb/core/tx/columnshard/engines/reader/read_filter_merger.cpp b/ydb/core/tx/columnshard/engines/reader/read_filter_merger.cpp
index 9540904a4cf..02ae3bfc2b4 100644
--- a/ydb/core/tx/columnshard/engines/reader/read_filter_merger.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/read_filter_merger.cpp
@@ -21,6 +21,7 @@ void TMergePartialStream::PutControlPoint(std::shared_ptr<TSortableBatchPosition
Y_VERIFY(point);
Y_VERIFY(point->IsSameSchema(SortSchema));
Y_VERIFY(++ControlPoints == 1);
+
SortHeap.emplace_back(TBatchIterator(*point));
if (SortHeap.size() > 1) {
Y_VERIFY(SortHeap.front().GetKeyColumns().Compare(SortHeap.back().GetKeyColumns()) != std::partial_ordering::greater);
diff --git a/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp
index 8ecf5fb4907..614b794f3a3 100644
--- a/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp
@@ -1,8 +1,10 @@
#include "read_metadata.h"
#include "order_control/default.h"
#include "order_control/pk_with_limit.h"
+#include <ydb/core/testlib/controllers/abstract.h>
#include <ydb/core/tx/columnshard/columnshard__index_scan.h>
#include <ydb/core/tx/columnshard/columnshard__stats_scan.h>
+#include <util/string/join.h>
namespace NKikimr::NOlap {
@@ -198,9 +200,9 @@ void TReadStats::PrintToLog() {
;
}
-NIndexedReader::IOrderPolicy::TPtr TReadMetadata::BuildSortingPolicy() const {
+NIndexedReader::IOrderPolicy::TPtr TReadMetadata::DoBuildSortingPolicy() const {
auto& indexInfo = ResultIndexSchema->GetIndexInfo();
- if (Limit && IsSorted() && indexInfo.IsSorted() && indexInfo.GetSortingKey()->num_fields() &&
+ if (Limit && IsSorted() && indexInfo.IsSorted() &&
indexInfo.GetReplaceKey()->Equals(indexInfo.GetIndexKey())) {
ui32 idx = 0;
for (auto&& i : indexInfo.GetPrimaryKey()) {
@@ -213,14 +215,19 @@ NIndexedReader::IOrderPolicy::TPtr TReadMetadata::BuildSortingPolicy() const {
++idx;
}
- if (!GetProgram().HasEarlyFilterOnly()) {
+ if (!idx || !GetProgram().HasEarlyFilterOnly()) {
return std::make_shared<NIndexedReader::TAnySorting>(this->shared_from_this());
}
-
return std::make_shared<NIndexedReader::TPKSortingWithLimit>(this->shared_from_this());
} else {
return std::make_shared<NIndexedReader::TAnySorting>(this->shared_from_this());
}
}
+std::shared_ptr<NIndexedReader::IOrderPolicy> TReadMetadata::BuildSortingPolicy() const {
+ auto result = DoBuildSortingPolicy();
+ NYDBTest::TControllers::GetColumnShardController()->OnSortingPolicy(result);
+ return result;
+}
+
}
diff --git a/ydb/core/tx/columnshard/engines/reader/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/read_metadata.h
index 6d93d765c66..2bc48226590 100644
--- a/ydb/core/tx/columnshard/engines/reader/read_metadata.h
+++ b/ydb/core/tx/columnshard/engines/reader/read_metadata.h
@@ -138,6 +138,7 @@ private:
std::shared_ptr<ISnapshotSchema> ResultIndexSchema;
std::vector<ui32> AllColumns;
std::vector<ui32> ResultColumnsIds;
+ std::shared_ptr<NIndexedReader::IOrderPolicy> DoBuildSortingPolicy() const;
public:
using TConstPtr = std::shared_ptr<const TReadMetadata>;
diff --git a/ydb/core/tx/columnshard/engines/storage/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule.cpp
index 3c39702af55..9eadbd77670 100644
--- a/ydb/core/tx/columnshard/engines/storage/granule.cpp
+++ b/ydb/core/tx/columnshard/engines/storage/granule.cpp
@@ -165,7 +165,7 @@ void TGranuleMeta::OnAdditiveSummaryChange() const {
if (AdditiveSummaryCache) {
Counters.OnCompactedData(AdditiveSummaryCache->GetOther());
Counters.OnInsertedData(AdditiveSummaryCache->GetInserted());
- Counters.OnFullData(AdditiveSummaryCache->GetGranuleSize(), AdditiveSummaryCache->GetActivePortionsCount());
+ Counters.OnFullData(AdditiveSummaryCache->GetOther() + AdditiveSummaryCache->GetInserted());
}
}
diff --git a/ydb/core/tx/columnshard/testlib/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/testlib/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..0457c3e18da
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(tx-columnshard-testlib)
+target_link_libraries(tx-columnshard-testlib PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ core-testlib-controllers
+ engines-reader-order_control
+)
+target_sources(tx-columnshard-testlib PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/testlib/controller.cpp
+)
diff --git a/ydb/core/tx/columnshard/testlib/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/testlib/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..ad0f10a58e0
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(tx-columnshard-testlib)
+target_link_libraries(tx-columnshard-testlib PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ core-testlib-controllers
+ engines-reader-order_control
+)
+target_sources(tx-columnshard-testlib PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/testlib/controller.cpp
+)
diff --git a/ydb/core/tx/columnshard/testlib/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/testlib/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..ad0f10a58e0
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(tx-columnshard-testlib)
+target_link_libraries(tx-columnshard-testlib PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ core-testlib-controllers
+ engines-reader-order_control
+)
+target_sources(tx-columnshard-testlib PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/testlib/controller.cpp
+)
diff --git a/ydb/core/tx/columnshard/testlib/CMakeLists.txt b/ydb/core/tx/columnshard/testlib/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/core/tx/columnshard/testlib/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/testlib/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..0457c3e18da
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(tx-columnshard-testlib)
+target_link_libraries(tx-columnshard-testlib PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ core-testlib-controllers
+ engines-reader-order_control
+)
+target_sources(tx-columnshard-testlib PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tx/columnshard/testlib/controller.cpp
+)
diff --git a/ydb/core/tx/columnshard/testlib/controller.cpp b/ydb/core/tx/columnshard/testlib/controller.cpp
new file mode 100644
index 00000000000..16e722803b4
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/controller.cpp
@@ -0,0 +1,18 @@
+#include "controller.h"
+#include <ydb/core/tx/columnshard/engines/reader/order_control/pk_with_limit.h>
+#include <ydb/core/tx/columnshard/engines/reader/order_control/default.h>
+
+namespace NKikimr::NYDBTest::NColumnShard {
+
+bool TController::DoOnSortingPolicy(std::shared_ptr<NOlap::NIndexedReader::IOrderPolicy> policy) {
+ if (dynamic_cast<const NOlap::NIndexedReader::TPKSortingWithLimit*>(policy.get())) {
+ ++SortingWithLimit;
+ } else if (dynamic_cast<const NOlap::NIndexedReader::TAnySorting*>(policy.get())) {
+ ++AnySorting;
+ } else {
+ Y_VERIFY(false);
+ }
+ return true;
+}
+
+}
diff --git a/ydb/core/tx/columnshard/testlib/controller.h b/ydb/core/tx/columnshard/testlib/controller.h
new file mode 100644
index 00000000000..edb4f7a7118
--- /dev/null
+++ b/ydb/core/tx/columnshard/testlib/controller.h
@@ -0,0 +1,18 @@
+#pragma once
+#include <ydb/core/testlib/controllers/abstract.h>
+
+namespace NKikimr::NYDBTest::NColumnShard {
+
+class TController: public ICSController {
+private:
+ YDB_READONLY(ui32, SortingWithLimit, 0);
+ YDB_READONLY(ui32, AnySorting, 0);
+protected:
+ virtual bool DoOnSortingPolicy(std::shared_ptr<NOlap::NIndexedReader::IOrderPolicy> policy);
+public:
+ bool HasPKSortingOnly() const {
+ return SortingWithLimit && !AnySorting;
+ }
+};
+
+}