diff options
author | eivanov89 <eivanov89@ydb.tech> | 2022-12-30 18:09:16 +0300 |
---|---|---|
committer | eivanov89 <eivanov89@ydb.tech> | 2022-12-30 18:09:16 +0300 |
commit | 08fb420704acef4bf445c80b67bc9f9fc648583f (patch) | |
tree | 9c806403ceca101998867e141d0e4a58973ea923 | |
parent | 58ff0f051f57461c1377ac11c38b0e1804fe9b4f (diff) | |
download | ydb-08fb420704acef4bf445c80b67bc9f9fc648583f.tar.gz |
move datashard load actors to common protocol load_test
47 files changed, 362 insertions, 711 deletions
diff --git a/ydb/core/base/services/datashard_service_id.h b/ydb/core/base/services/datashard_service_id.h deleted file mode 100644 index b1526ad7e61..00000000000 --- a/ydb/core/base/services/datashard_service_id.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "defs.h" -#include <util/generic/string.h> - -namespace NKikimr { - -inline TActorId MakeDataShardLoadId(ui32 nodeId) { - char x[12] = {'d', 's', 'l', 'o', 'a', 'd', 'd', 0}; - x[8] = (char)(nodeId >> 24); - x[9] = (char)(nodeId >> 16); - x[10] = (char)(nodeId >> 8); - x[11] = (char)nodeId; - return TActorId(nodeId, TStringBuf(x, 12)); -} - -} // NKikimr diff --git a/ydb/core/client/server/CMakeLists.darwin.txt b/ydb/core/client/server/CMakeLists.darwin.txt index 6091df111aa..98a82a55e31 100644 --- a/ydb/core/client/server/CMakeLists.darwin.txt +++ b/ydb/core/client/server/CMakeLists.darwin.txt @@ -75,7 +75,6 @@ target_sources(core-client-server PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_resolve_node.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_ic_debug.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_load.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_datashard_load.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_minikql.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_scheme_tx.cpp diff --git a/ydb/core/client/server/CMakeLists.linux-aarch64.txt b/ydb/core/client/server/CMakeLists.linux-aarch64.txt index 8f33a89e5b8..58f800317e7 100644 --- a/ydb/core/client/server/CMakeLists.linux-aarch64.txt +++ b/ydb/core/client/server/CMakeLists.linux-aarch64.txt @@ -76,7 +76,6 @@ target_sources(core-client-server PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_resolve_node.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_ic_debug.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_load.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_datashard_load.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_minikql.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_scheme_tx.cpp diff --git a/ydb/core/client/server/CMakeLists.linux.txt b/ydb/core/client/server/CMakeLists.linux.txt index 8f33a89e5b8..58f800317e7 100644 --- a/ydb/core/client/server/CMakeLists.linux.txt +++ b/ydb/core/client/server/CMakeLists.linux.txt @@ -76,7 +76,6 @@ target_sources(core-client-server PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_resolve_node.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_ic_debug.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_load.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_datashard_load.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_minikql.cpp ${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_local_scheme_tx.cpp diff --git a/ydb/core/client/server/grpc_server.cpp b/ydb/core/client/server/grpc_server.cpp index 65da74ab04e..457ad85af29 100644 --- a/ydb/core/client/server/grpc_server.cpp +++ b/ydb/core/client/server/grpc_server.cpp @@ -485,7 +485,6 @@ void TGRpcService::SetupIncomingRequests() { ADD_ACTOR_REQUEST(TabletKillRequest, TTabletKillRequest, MTYPE_CLIENT_TABLET_KILL_REQUEST) ADD_ACTOR_REQUEST(SchemeOperationStatus, TSchemeOperationStatus, MTYPE_CLIENT_FLAT_TX_STATUS_REQUEST) ADD_ACTOR_REQUEST(BlobStorageLoadRequest, TBsTestLoadRequest, MTYPE_CLIENT_LOAD_REQUEST) - ADD_ACTOR_REQUEST(DataShardLoadRequest, TDsTestLoadRequest, MTYPE_CLIENT_DS_LOAD_REQUEST) ADD_ACTOR_REQUEST(BlobStorageGetRequest, TBsGetRequest, MTYPE_CLIENT_GET_REQUEST) ADD_ACTOR_REQUEST(ChooseProxy, TChooseProxyRequest, MTYPE_CLIENT_CHOOSE_PROXY) ADD_ACTOR_REQUEST(WhoAmI, TWhoAmI, MTYPE_CLIENT_WHOAMI) diff --git a/ydb/core/client/server/msgbus_server.cpp b/ydb/core/client/server/msgbus_server.cpp index 80b23170847..2337dd6f8c3 100644 --- a/ydb/core/client/server/msgbus_server.cpp +++ b/ydb/core/client/server/msgbus_server.cpp @@ -116,8 +116,6 @@ public: MTYPE(TBusSchemeDescribe) MTYPE(TBusOldFlatDescribeRequest) MTYPE(TBusOldFlatDescribeResponse) - MTYPE(TBusDsTestLoadRequest) - MTYPE(TBusDsTestLoadResponse) MTYPE(TBusBsTestLoadRequest) MTYPE(TBusBsTestLoadResponse) MTYPE(TBusBsGetRequest) @@ -178,7 +176,6 @@ public: REPLY_OPTION(TBusResponse) REPLY_OPTION(TBusDbResponse) - REPLY_OPTION(TBusDsTestLoadResponse) REPLY_OPTION(TBusBsTestLoadResponse) REPLY_OPTION(TBusNodeRegistrationResponse) REPLY_OPTION(TBusCmsResponse) @@ -545,8 +542,6 @@ void TMessageBusServer::OnMessage(TBusMessageContext &msg) { case MTYPE_CLIENT_FLAT_DESCRIBE_REQUEST: case MTYPE_CLIENT_OLD_FLAT_DESCRIBE_REQUEST: return ClientProxyRequest<TEvBusProxy::TEvFlatDescribeRequest>(msg); - case MTYPE_CLIENT_DS_LOAD_REQUEST: - return ClientActorRequest(CreateMessageBusDataShardLoadRequest, msg); case MTYPE_CLIENT_LOAD_REQUEST: return ClientActorRequest(CreateMessageBusBlobStorageLoadRequest, msg); case MTYPE_CLIENT_GET_REQUEST: diff --git a/ydb/core/client/server/msgbus_server.h b/ydb/core/client/server/msgbus_server.h index b554bce62df..0e6da0ecbb3 100644 --- a/ydb/core/client/server/msgbus_server.h +++ b/ydb/core/client/server/msgbus_server.h @@ -291,7 +291,6 @@ IActor* CreateMessageBusChooseProxy(TBusMessageContext &msg); IActor* CreateMessageBusTabletStateRequest(TBusMessageContext &msg); IActor* CreateMessageBusTabletKillRequest(TBusMessageContext &msg); IActor* CreateMessageBusSchemeOperationStatus(TBusMessageContext &msg); -IActor* CreateMessageBusDataShardLoadRequest(TBusMessageContext &msg); IActor* CreateMessageBusBlobStorageLoadRequest(TBusMessageContext &msg); IActor* CreateMessageBusBlobStorageGetRequest(TBusMessageContext &msg); IActor* CreateMessageBusHiveLookupTablet(TBusMessageContext &msg); diff --git a/ydb/core/client/server/msgbus_server_datashard_load.cpp b/ydb/core/client/server/msgbus_server_datashard_load.cpp deleted file mode 100644 index ac7b1cc32e4..00000000000 --- a/ydb/core/client/server/msgbus_server_datashard_load.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "msgbus_servicereq.h" - -#include <ydb/core/base/services/datashard_service_id.h> -#include <ydb/core/tx/datashard/testload/test_load_actor.h> - -namespace NKikimr::NMsgBusProxy { - -class TDsTestLoadActorRequest : public TActorBootstrapped<TDsTestLoadActorRequest>, public TMessageBusSessionIdentHolder { - ui32 NodeId = 0; - NKikimrDataShardLoad::TEvTestLoadRequest Cmd; - NKikimrClient::TDsTestLoadResponse Response; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::MSGBUS_COMMON; - } - - TDsTestLoadActorRequest(NKikimrClient::TDsTestLoadRequest& record, NMsgBusProxy::TBusMessageContext& msg) - : TMessageBusSessionIdentHolder(msg) - , NodeId(record.GetNodeId()) - , Cmd(record.GetEvent()) - {} - - void Bootstrap(const TActorContext& ctx) { - auto msg = std::make_unique<TEvDataShardLoad::TEvTestLoadRequest>(); - msg->Record = Cmd; - msg->Record.SetCookie(NodeId); - ctx.Send(MakeDataShardLoadId(NodeId), msg.release()); - - Become(&TDsTestLoadActorRequest::StateFunc); - } - - void Handle(TEvDataShardLoad::TEvTestLoadResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - ui32 nodeId = record.GetCookie(); - - NKikimrClient::TDsTestLoadResponse::TItem *item = Response.AddItems(); - item->SetNodeId(nodeId); - if (record.HasStatus()) { - item->SetStatus(record.GetStatus()); - } - if (record.HasErrorReason()) { - item->SetErrorReason(record.GetErrorReason()); - } - - auto response = MakeHolder<TBusDsTestLoadResponse>(); - response->Record = Response; - SendReplyMove(response.Release()); - Die(ctx); - } - - STFUNC(StateFunc) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvDataShardLoad::TEvTestLoadResponse, Handle); - } - } -}; - -IActor *CreateMessageBusDataShardLoadRequest(NMsgBusProxy::TBusMessageContext& msg) { - NKikimrClient::TDsTestLoadRequest& record = static_cast<TBusDsTestLoadRequest *>(msg.GetMessage())->Record; - return new TDsTestLoadActorRequest(record, msg); -} - -} // NKikimr::NMsgBusProxy diff --git a/ydb/core/driver_lib/cli_utils/cli.h b/ydb/core/driver_lib/cli_utils/cli.h index 02fc9b40fe6..e7a87c8f294 100644 --- a/ydb/core/driver_lib/cli_utils/cli.h +++ b/ydb/core/driver_lib/cli_utils/cli.h @@ -36,7 +36,6 @@ namespace NDriverClient { int PersQueueStress(TCommandConfig &cmdConf, int argc, char **argv); int PersQueueDiscoverClustersRequest(TCommandConfig &cmdConf, int argc, char **argv); int LoadRequest(TCommandConfig &cmdConf, int argc, char **argv); - int DsLoadRequest(TCommandConfig &cmdConf, int argc, char **argv); int ActorsysPerfTest(TCommandConfig &cmdConf, int argc, char **argv); void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix); void HideOptions(NLastGetopt::TOpts& opts); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmd_config.h b/ydb/core/driver_lib/cli_utils/cli_cmd_config.h index 651da5b99af..bb0b69dbcb6 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmd_config.h +++ b/ydb/core/driver_lib/cli_utils/cli_cmd_config.h @@ -36,13 +36,6 @@ namespace NDriverClient { break; } - case NMsgBusProxy::MTYPE_CLIENT_DS_LOAD_RESPONSE: { - TAutoPtr<NMsgBusProxy::TBusDsTestLoadResponse> x(new NMsgBusProxy::TBusDsTestLoadResponse); - x->Record = data.template GetResult<NKikimrClient::TDsTestLoadResponse>(); - response = x.Release(); - break; - } - default: Y_FAIL("unexpected reply message type"); } diff --git a/ydb/core/driver_lib/cli_utils/cli_load.cpp b/ydb/core/driver_lib/cli_utils/cli_load.cpp index d6dc154533f..c2567f5e4f9 100644 --- a/ydb/core/driver_lib/cli_utils/cli_load.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_load.cpp @@ -59,45 +59,5 @@ int LoadRequest(TCommandConfig& /*cmdConf*/, int argc, char **argv) { return EXIT_SUCCESS; } -int DsLoadRequest(TCommandConfig& /*cmdConf*/, int argc, char** argv) { - TCmdLoadConfig config; - config.Parse(argc, argv); - - TAutoPtr<NMsgBusProxy::TBusDsTestLoadRequest> request(new NMsgBusProxy::TBusDsTestLoadRequest); - const bool isOk = ::google::protobuf::TextFormat::ParseFromString(config.Proto, &request->Record); - if (!isOk) { - ythrow TWithBackTrace<yexception>() << "Error parsing protobuf: '" << config.Proto << "'"; - } - - TAutoPtr<NBus::TBusMessage> reply; - NBus::EMessageStatus status = config.SyncCall(request, reply); - - switch (status) { - case NBus::MESSAGE_OK: { - const NKikimrClient::TDsTestLoadResponse& response = - static_cast<NMsgBusProxy::TBusDsTestLoadResponse *>(reply.Get())->Record; - bool status = true; - for (const NKikimrClient::TDsTestLoadResponse::TItem& item : response.GetItems()) { - if (item.GetStatus() != NMsgBusProxy::MSTATUS_OK) { - Cerr << "NodeId# " << item.GetNodeId() << " ErrorReason# " << item.GetErrorReason() << Endl; - status = false; - } - } - if (!status) { - return EXIT_FAILURE; - } - break; - } - - default: { - const char *description = NBus::MessageStatusDescription(status); - Cerr << description << Endl; - return EXIT_FAILURE; - } - } - - return EXIT_SUCCESS; -} - } // NKikimr } // NDriverClient diff --git a/ydb/core/driver_lib/run/driver.h b/ydb/core/driver_lib/run/driver.h index 5de1374559e..7391e866d66 100644 --- a/ydb/core/driver_lib/run/driver.h +++ b/ydb/core/driver_lib/run/driver.h @@ -36,7 +36,6 @@ namespace NKikimr { XX(EDM_PERSQUEUE_STRESS, "persqueue-stress", "stress read or write to a persqueue tablet") \ XX(EDM_PERSQUEUE_DISCOVER_CLUSTERS, "persqueue-discover-clusters", "persqueue session clusters discovery") \ XX(EDM_LOAD_REQUEST, "load-test", "send protobuf request to test load actor") \ - XX(EDM_DS_LOAD_REQUEST, "ds-load-test", "send protobuf request to datashard test load actor (https://wiki.yandex-team.ru/kikimr/developers/DSLoadTest/)") \ XX(EDM_ACTORSYS_PERFTEST, "actorsys-perf-test", "make actorsystem performance test") \ diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index f0b3f56384f..9cf6e472ce5 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -22,7 +22,6 @@ #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/base/tabletid.h> #include <ydb/core/base/user_registry.h> -#include <ydb/core/base/services/datashard_service_id.h> #include <ydb/core/blobstorage/backpressure/unisched.h> #include <ydb/core/blobstorage/nodewarden/node_warden.h> @@ -118,7 +117,6 @@ #include <ydb/core/tx/coordinator/coordinator.h> #include <ydb/core/tx/columnshard/blob_cache.h> #include <ydb/core/tx/datashard/datashard.h> -#include <ydb/core/tx/datashard/testload/test_load_actor.h> #include <ydb/core/tx/columnshard/columnshard.h> #include <ydb/core/tx/mediator/mediator.h> #include <ydb/core/tx/replication/controller/controller.h> @@ -1858,9 +1856,6 @@ void TLoadInitializer::InitializeServices(NActors::TActorSystemSetup *setup, con IActor *bsActor = CreateLoadTestActor(appData->Counters); setup->LocalServices.emplace_back(MakeLoadServiceID(NodeId), TActorSetupCmd(bsActor, TMailboxType::HTSwap, appData->UserPoolId)); // FIXME: correct service id - - IActor *dsActor = NDataShardLoad::CreateTestLoadActor(appData->Counters); - setup->LocalServices.emplace_back(MakeDataShardLoadId(NodeId), TActorSetupCmd(dsActor, TMailboxType::HTSwap, appData->UserPoolId)); } // TFailureInjectionInitializer diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp index 8dce0f34a25..8d41149a8f6 100644 --- a/ydb/core/driver_lib/run/main.cpp +++ b/ydb/core/driver_lib/run/main.cpp @@ -148,8 +148,6 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> return NDriverClient::PersQueueDiscoverClustersRequest(cmdConf, argc, argv); case EDM_LOAD_REQUEST: return NDriverClient::LoadRequest(cmdConf, argc, argv); - case EDM_DS_LOAD_REQUEST: - return NDriverClient::DsLoadRequest(cmdConf, argc, argv); case EDM_ACTORSYS_PERFTEST: return NDriverClient::ActorsysPerfTest(cmdConf, argc, argv); default: diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index c35be9b29ef..bf343a76a16 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -43,7 +43,6 @@ #include <ydb/core/base/counters.h> #include <ydb/core/base/tabletid.h> #include <ydb/core/base/statestorage_impl.h> -#include <ydb/core/base/services/datashard_service_id.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/mind/local.h> @@ -1220,14 +1219,6 @@ void TKikimrRunner::InitializeActorSystem( false, ActorSystem.Get(), MakeLoadServiceID(runConfig.NodeId)); - - Monitoring->RegisterActorPage( - ActorsMonPage, - "dsload", - "DSLoad", - false, - ActorSystem.Get(), - MakeDataShardLoadId(runConfig.NodeId)); } if (servicesMask.EnableFailureInjectionService) { diff --git a/ydb/core/load_test/CMakeLists.darwin.txt b/ydb/core/load_test/CMakeLists.darwin.txt index 21fca4b64df..478623a5321 100644 --- a/ydb/core/load_test/CMakeLists.darwin.txt +++ b/ydb/core/load_test/CMakeLists.darwin.txt @@ -6,13 +6,14 @@ # original buildsystem will not be accepted. +add_subdirectory(ut_ycsb) add_library(ydb-core-load_test) target_link_libraries(ydb-core-load_test PUBLIC contrib-libs-cxxsupp yutil - cpp-histogram-hdr contrib-libs-protobuf + cpp-histogram-hdr monlib-dynamic_counters-percentile monlib-service-pages ydb-core-base @@ -36,4 +37,9 @@ target_sources(ydb-core-load_test PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/load_test/service_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/load_test/group_write.cpp ${CMAKE_SOURCE_DIR}/ydb/core/load_test/vdisk_write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/bulk_mkql_upsert.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/info_collector.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/kqp_upsert.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/test_load_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/test_load_read_iterator.cpp ) diff --git a/ydb/core/load_test/CMakeLists.linux-aarch64.txt b/ydb/core/load_test/CMakeLists.linux-aarch64.txt index 84ef7985971..598df6875ec 100644 --- a/ydb/core/load_test/CMakeLists.linux-aarch64.txt +++ b/ydb/core/load_test/CMakeLists.linux-aarch64.txt @@ -6,14 +6,15 @@ # original buildsystem will not be accepted. +add_subdirectory(ut_ycsb) add_library(ydb-core-load_test) target_link_libraries(ydb-core-load_test PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - cpp-histogram-hdr contrib-libs-protobuf + cpp-histogram-hdr monlib-dynamic_counters-percentile monlib-service-pages ydb-core-base @@ -37,4 +38,9 @@ target_sources(ydb-core-load_test PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/load_test/service_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/load_test/group_write.cpp ${CMAKE_SOURCE_DIR}/ydb/core/load_test/vdisk_write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/bulk_mkql_upsert.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/info_collector.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/kqp_upsert.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/test_load_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/test_load_read_iterator.cpp ) diff --git a/ydb/core/load_test/CMakeLists.linux.txt b/ydb/core/load_test/CMakeLists.linux.txt index 84ef7985971..598df6875ec 100644 --- a/ydb/core/load_test/CMakeLists.linux.txt +++ b/ydb/core/load_test/CMakeLists.linux.txt @@ -6,14 +6,15 @@ # original buildsystem will not be accepted. +add_subdirectory(ut_ycsb) add_library(ydb-core-load_test) target_link_libraries(ydb-core-load_test PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - cpp-histogram-hdr contrib-libs-protobuf + cpp-histogram-hdr monlib-dynamic_counters-percentile monlib-service-pages ydb-core-base @@ -37,4 +38,9 @@ target_sources(ydb-core-load_test PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/load_test/service_actor.cpp ${CMAKE_SOURCE_DIR}/ydb/core/load_test/group_write.cpp ${CMAKE_SOURCE_DIR}/ydb/core/load_test/vdisk_write.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/bulk_mkql_upsert.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/info_collector.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/kqp_upsert.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/test_load_actor.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ycsb/test_load_read_iterator.cpp ) diff --git a/ydb/core/load_test/events.h b/ydb/core/load_test/events.h index 05bc60ce88e..f3cfea6d58b 100644 --- a/ydb/core/load_test/events.h +++ b/ydb/core/load_test/events.h @@ -79,7 +79,7 @@ struct TEvLoad { TString LastHtmlPage; NJson::TJsonValue JsonResult; - TEvLoadTestFinished(ui64 tag, TIntrusivePtr<TLoadReport> report, TString errorReason) + TEvLoadTestFinished(ui64 tag, TIntrusivePtr<TLoadReport> report, TString errorReason = {}) : Tag(tag) , Report(report) , ErrorReason(errorReason) diff --git a/ydb/core/load_test/service_actor.cpp b/ydb/core/load_test/service_actor.cpp index 90642449acf..52142328383 100644 --- a/ydb/core/load_test/service_actor.cpp +++ b/ydb/core/load_test/service_actor.cpp @@ -4,6 +4,7 @@ #include <ydb/core/base/counters.h> #include <ydb/public/lib/base/msgbus.h> #include <ydb/core/blobstorage/base/blobstorage_events.h> +#include <ydb/core/load_test/ycsb/test_load_actor.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/json/json_writer.h> @@ -253,6 +254,19 @@ public: break; } + case NKikimr::TEvLoadTestRequest::CommandCase::kYCSBLoad: { + const auto& cmd = record.GetYCSBLoad(); + tag = GetOrGenerateTag(cmd); + if (LoadActors.count(tag) != 0) { + ythrow TLoadActorException() << Sprintf("duplicate load actor with Tag# %" PRIu64, tag); + } + + LOG_D("Create new YCSB load actor with tag# " << tag); + LoadActors.emplace(tag, TlsActivationContext->Register(NDataShardLoad::CreateTestLoadActor( + cmd, SelfId(), GetServiceCounters(Counters, "load_actor"), tag))); + break; + } + default: { TString protoTxt; google::protobuf::TextFormat::PrintToString(record, &protoTxt); diff --git a/ydb/core/tx/datashard/datashard_ut_testload.cpp b/ydb/core/load_test/ut_ycsb.cpp index 4ee4b073c72..b7c0d557e5e 100644 --- a/ydb/core/tx/datashard/datashard_ut_testload.cpp +++ b/ydb/core/load_test/ut_ycsb.cpp @@ -1,11 +1,13 @@ -#include "datashard_ut_common.h" - #include <ydb/core/kqp/ut/common/kqp_ut_common.h> // Y_UNIT_TEST_(TWIN|QUAD), Q_ +#include <ydb/core/load_test/events.h> +#include <ydb/core/load_test/ycsb/test_load_actor.h> +#include <ydb/core/scheme/scheme_types_defs.h> +#include <ydb/core/scheme/scheme_types_proto.h> +#include <ydb/core/tx/schemeshard/schemeshard.h> +#include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/library/yql/minikql/mkql_node_printer.h> -#include "testload/test_load_actor.h" - namespace NKikimr { using namespace NKikimr::NDataShardLoad; @@ -25,26 +27,143 @@ TString GetKey(size_t n) { return Sprintf("user%.19lu", n); } +void InitRoot(Tests::TServer::TPtr server, + TActorId sender) { + server->SetupRootStoragePools(sender); +} + +ui64 RunSchemeTx( + TTestActorRuntimeBase& runtime, + THolder<TEvTxUserProxy::TEvProposeTransaction>&& request, + TActorId sender = {}, + bool viaActorSystem = false, + TEvTxUserProxy::TEvProposeTransactionStatus::EStatus expectedStatus = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress) +{ + if (!sender) { + sender = runtime.AllocateEdgeActor(); + } + + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release()), 0, viaActorSystem); + auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), expectedStatus); + + return ev->Get()->Record.GetTxId(); +} + +THolder<TEvTxUserProxy::TEvProposeTransaction> SchemeTxTemplate( + NKikimrSchemeOp::EOperationType type, + const TString& workingDir = {}) +{ + auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + request->Record.SetExecTimeoutPeriod(Max<ui64>()); + + auto& tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + tx.SetOperationType(type); + + if (workingDir) { + tx.SetWorkingDir(workingDir); + } + + return request; +} + +void WaitTxNotification(Tests::TServer::TPtr server, TActorId sender, ui64 txId) { + auto &runtime = *server->GetRuntime(); + auto &settings = server->GetSettings(); + + auto request = MakeHolder<NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion>(); + request->Record.SetTxId(txId); + auto tid = ChangeStateStorage(SchemeRoot, settings.Domain); + runtime.SendToPipe(tid, sender, request.Release(), 0, GetPipeConfigWithRetries()); + runtime.GrabEdgeEventRethrow<TEvSchemeShard::TEvNotifyTxCompletionResult>(sender); +} + void CreateTable(Tests::TServer::TPtr server, TActorId sender, const TString &root, const TString& tableName) { - TVector<TShardedTableOptions::TColumn> columns; - columns.reserve(ValueColumnsCount + 1); + auto request = SchemeTxTemplate(NKikimrSchemeOp::ESchemeOpCreateTable, root); + + auto& tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + tx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateTable); + NKikimrSchemeOp::TTableDescription* desc = tx.MutableCreateTable(); - columns.emplace_back("id", "Utf8", true, false); + UNIT_ASSERT(desc); + desc->SetName(tableName); + desc->SetUniformPartitionsCount(1); + + { + auto col = desc->AddColumns(); + col->SetName("id"); + col->SetType("Utf8"); + col->SetNotNull(true); + desc->AddKeyColumnNames("id"); + } for (size_t i = 0; i < ValueColumnsCount; ++i) { TString fieldName = FieldPrefix + ToString(i); - columns.emplace_back(fieldName, "String", false, false); + auto col = desc->AddColumns(); + col->SetName(fieldName); + col->SetType("String"); + col->SetNotNull(false); } - auto opts = TShardedTableOptions() - .Shards(1) - .Columns(columns); + WaitTxNotification(server, sender, RunSchemeTx(*server->GetRuntime(), std::move(request), sender)); +} + +NKikimrScheme::TEvDescribeSchemeResult DescribeTable(Tests::TServer::TPtr server, + TActorId sender, + const TString &path) +{ + auto &runtime = *server->GetRuntime(); + TAutoPtr<IEventHandle> handle; + TVector<ui64> shards; + + auto request = MakeHolder<TEvTxUserProxy::TEvNavigate>(); + request->Record.MutableDescribePath()->SetPath(path); + request->Record.MutableDescribePath()->MutableOptions()->SetShowPrivateTable(true); + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); + auto reply = runtime.GrabEdgeEventRethrow<TEvSchemeShard::TEvDescribeSchemeResult>(handle); + + return *reply->MutableRecord(); +} + +TVector<ui64> GetTableShards(Tests::TServer::TPtr server, + TActorId sender, + const TString &path) +{ + TVector<ui64> shards; + auto lsResult = DescribeTable(server, sender, path); + for (auto &part : lsResult.GetPathDescription().GetTablePartitions()) + shards.push_back(part.GetDatashardId()); + + return shards; +} + +using TTableInfoMap = THashMap<TString, NKikimrTxDataShard::TEvGetInfoResponse::TUserTable>; + +std::pair<TTableInfoMap, ui64> GetTables( + Tests::TServer::TPtr server, + ui64 tabletId) +{ + auto &runtime = *server->GetRuntime(); + + auto sender = runtime.AllocateEdgeActor(); + auto request = MakeHolder<TEvDataShard::TEvGetInfoRequest>(); + runtime.SendToPipe(tabletId, sender, request.Release(), 0, GetPipeConfigWithRetries()); + + TTableInfoMap result; + + TAutoPtr<IEventHandle> handle; + auto response = runtime.GrabEdgeEventRethrow<TEvDataShard::TEvGetInfoResponse>(handle); + for (auto& table: response->Record.GetUserTables()) { + result[table.GetName()] = table; + } + + auto ownerId = response->Record.GetTabletInfo().GetSchemeShard(); - CreateShardedTable(server, sender, root, tableName, opts); + return std::make_pair(result, ownerId); } TVector<TCell> ToCells(const std::vector<TString>& keys) { @@ -215,41 +334,28 @@ struct TTestHelper { } } - std::unique_ptr<TEvDataShardLoad::TEvTestLoadFinished> RunTestLoad( - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request) + std::unique_ptr<TEvLoad::TEvLoadTestFinished> RunTestLoad( + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request) { request->Record.SetNotifyWhenFinished(true); auto &runtime = *Server->GetRuntime(); TIntrusivePtr<::NMonitoring::TDynamicCounters> counters(new ::NMonitoring::TDynamicCounters()); - auto testLoadActor = runtime.Register(CreateTestLoadActor(counters)); - - runtime.Send(new IEventHandle(testLoadActor, Sender, request.release()), 0, true); - - { - // check load started - TAutoPtr<IEventHandle> handle; - runtime.GrabEdgeEventRethrow<TEvDataShardLoad::TEvTestLoadResponse>(handle); - UNIT_ASSERT(handle); - auto response = handle->Release<TEvDataShardLoad::TEvTestLoadResponse>(); - auto& responseRecord = response->Record; - UNIT_ASSERT_VALUES_EQUAL(responseRecord.GetStatus(), NMsgBusProxy::MSTATUS_OK); - } + runtime.Register(CreateTestLoadActor(request->Record, Sender, counters, 0)); { // wait until load finished TAutoPtr<IEventHandle> handle; - runtime.GrabEdgeEventRethrow<TEvDataShardLoad::TEvTestLoadFinished>(handle); + runtime.GrabEdgeEventRethrow<TEvLoad::TEvLoadTestFinished>(handle); UNIT_ASSERT(handle); - auto response = handle->Release<TEvDataShardLoad::TEvTestLoadFinished>(); - UNIT_ASSERT(response->Record.HasReport()); - UNIT_ASSERT(!response->Record.HasErrorReason()); + auto response = handle->Release<TEvLoad::TEvLoadTestFinished>(); + UNIT_ASSERT(response->ErrorReason.Empty()); - return std::unique_ptr<TEvDataShardLoad::TEvTestLoadFinished>(response.Release()); + return std::unique_ptr<TEvLoad::TEvLoadTestFinished>(response.Release()); } } void RunUpsertTestLoad( - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> loadRequest, + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> loadRequest, size_t keyFrom, size_t expectedRowCount, bool forceResolve = false) @@ -277,7 +383,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -298,7 +404,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 100; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -321,7 +427,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -343,7 +449,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 keyFrom = 12345; const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -364,7 +470,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertLocalMkqlStart(); @@ -386,7 +492,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertLocalMkqlStart(); @@ -408,7 +514,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 keyFrom = 12345; const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertLocalMkqlStart(); @@ -429,7 +535,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 20; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertKqpStart(); @@ -452,7 +558,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 20; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertKqpStart(); @@ -474,7 +580,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 keyFrom = 12345; const ui64 expectedRowCount = 20; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertKqpStart(); @@ -497,7 +603,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -529,7 +635,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { { // write some data, which should not be seen after drop - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -546,7 +652,7 @@ Y_UNIT_TEST_SUITE(UpsertLoad) { // because of drop we should see only these rows const ui64 expectedRowCount = 10; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableUpsertBulkStart(); @@ -577,7 +683,7 @@ Y_UNIT_TEST_SUITE(ReadLoad) { const ui64 expectedRowCount = 1000; - std::unique_ptr<TEvDataShardLoad::TEvTestLoadRequest> request(new TEvDataShardLoad::TEvTestLoadRequest()); + std::unique_ptr<TEvDataShardLoad::TEvYCSBTestLoadRequest> request(new TEvDataShardLoad::TEvYCSBTestLoadRequest()); auto& record = request->Record; auto& command = *record.MutableReadIteratorStart(); @@ -593,10 +699,9 @@ Y_UNIT_TEST_SUITE(ReadLoad) { setupTable.SetTableName("usertable"); auto result = helper.RunTestLoad(std::move(request)); - UNIT_ASSERT(result->Record.HasReport()); - UNIT_ASSERT_VALUES_EQUAL(result->Record.GetReport().GetSubtestCount(), 4); - UNIT_ASSERT_VALUES_EQUAL(result->Record.GetReport().GetOperationsOK(), (4 * expectedRowCount)); + UNIT_ASSERT_VALUES_EQUAL(result->JsonResult["subtests"].GetInteger(), 4); + UNIT_ASSERT_VALUES_EQUAL(result->JsonResult["oks"].GetInteger(), (4 * expectedRowCount)); // sanity check that there was data in table helper.CheckKeys(0, expectedRowCount); diff --git a/ydb/core/tx/datashard/ut_testload/CMakeLists.darwin.txt b/ydb/core/load_test/ut_ycsb/CMakeLists.darwin.txt index b81f59a7042..ec0116a041d 100644 --- a/ydb/core/tx/datashard/ut_testload/CMakeLists.darwin.txt +++ b/ydb/core/load_test/ut_ycsb/CMakeLists.darwin.txt @@ -7,19 +7,19 @@ -add_executable(ydb-core-tx-datashard-ut_testload) -target_compile_options(ydb-core-tx-datashard-ut_testload PRIVATE +add_executable(ydb-core-load_test-ut_ycsb) +target_compile_options(ydb-core-load_test-ut_ycsb PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-datashard-ut_testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard +target_include_directories(ydb-core-load_test-ut_ycsb PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/load_test ) -target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC +target_link_libraries(ydb-core-load_test-ut_ycsb PUBLIC contrib-libs-cxxsupp yutil library-cpp-cpuid_check cpp-testing-unittest_main - core-tx-datashard + ydb-core-load_test library-cpp-getopt cpp-regex-pcre library-cpp-svnversion @@ -30,7 +30,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC public-lib-yson_value cpp-client-ydb_result ) -target_link_options(ydb-core-tx-datashard-ut_testload PRIVATE +target_link_options(ydb-core-load_test-ut_ycsb PRIVATE -Wl,-no_deduplicate -Wl,-sdk_version,10.15 -fPIC @@ -38,19 +38,18 @@ target_link_options(ydb-core-tx-datashard-ut_testload PRIVATE -framework CoreFoundation ) -target_sources(ydb-core-tx-datashard-ut_testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/datashard_ut_common.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/datashard_ut_testload.cpp +target_sources(ydb-core-load_test-ut_ycsb PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ut_ycsb.cpp ) add_test( NAME - ydb-core-tx-datashard-ut_testload + ydb-core-load_test-ut_ycsb COMMAND - ydb-core-tx-datashard-ut_testload + ydb-core-load_test-ut_ycsb --print-before-suite --print-before-test --fork-tests --print-times --show-fails ) -vcs_info(ydb-core-tx-datashard-ut_testload) +vcs_info(ydb-core-load_test-ut_ycsb) diff --git a/ydb/core/tx/datashard/ut_testload/CMakeLists.linux-aarch64.txt b/ydb/core/load_test/ut_ycsb/CMakeLists.linux-aarch64.txt index efcbf2d674d..58d76a01210 100644 --- a/ydb/core/tx/datashard/ut_testload/CMakeLists.linux-aarch64.txt +++ b/ydb/core/load_test/ut_ycsb/CMakeLists.linux-aarch64.txt @@ -7,20 +7,20 @@ -add_executable(ydb-core-tx-datashard-ut_testload) -target_compile_options(ydb-core-tx-datashard-ut_testload PRIVATE +add_executable(ydb-core-load_test-ut_ycsb) +target_compile_options(ydb-core-load_test-ut_ycsb PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-datashard-ut_testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard +target_include_directories(ydb-core-load_test-ut_ycsb PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/load_test ) -target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC +target_link_libraries(ydb-core-load_test-ut_ycsb PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil library-cpp-lfalloc cpp-testing-unittest_main - core-tx-datashard + ydb-core-load_test library-cpp-getopt cpp-regex-pcre library-cpp-svnversion @@ -31,7 +31,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC public-lib-yson_value cpp-client-ydb_result ) -target_link_options(ydb-core-tx-datashard-ut_testload PRIVATE +target_link_options(ydb-core-load_test-ut_ycsb PRIVATE -ldl -lrt -Wl,--no-as-needed @@ -41,19 +41,18 @@ target_link_options(ydb-core-tx-datashard-ut_testload PRIVATE -lrt -ldl ) -target_sources(ydb-core-tx-datashard-ut_testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/datashard_ut_common.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/datashard_ut_testload.cpp +target_sources(ydb-core-load_test-ut_ycsb PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ut_ycsb.cpp ) add_test( NAME - ydb-core-tx-datashard-ut_testload + ydb-core-load_test-ut_ycsb COMMAND - ydb-core-tx-datashard-ut_testload + ydb-core-load_test-ut_ycsb --print-before-suite --print-before-test --fork-tests --print-times --show-fails ) -vcs_info(ydb-core-tx-datashard-ut_testload) +vcs_info(ydb-core-load_test-ut_ycsb) diff --git a/ydb/core/tx/datashard/ut_testload/CMakeLists.linux.txt b/ydb/core/load_test/ut_ycsb/CMakeLists.linux.txt index bc94a130f48..ad2a3c7a136 100644 --- a/ydb/core/tx/datashard/ut_testload/CMakeLists.linux.txt +++ b/ydb/core/load_test/ut_ycsb/CMakeLists.linux.txt @@ -7,14 +7,14 @@ -add_executable(ydb-core-tx-datashard-ut_testload) -target_compile_options(ydb-core-tx-datashard-ut_testload PRIVATE +add_executable(ydb-core-load_test-ut_ycsb) +target_compile_options(ydb-core-load_test-ut_ycsb PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-datashard-ut_testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard +target_include_directories(ydb-core-load_test-ut_ycsb PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/load_test ) -target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC +target_link_libraries(ydb-core-load_test-ut_ycsb PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC libs-tcmalloc-no_percpu_cache library-cpp-cpuid_check cpp-testing-unittest_main - core-tx-datashard + ydb-core-load_test library-cpp-getopt cpp-regex-pcre library-cpp-svnversion @@ -33,7 +33,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC public-lib-yson_value cpp-client-ydb_result ) -target_link_options(ydb-core-tx-datashard-ut_testload PRIVATE +target_link_options(ydb-core-load_test-ut_ycsb PRIVATE -ldl -lrt -Wl,--no-as-needed @@ -43,19 +43,18 @@ target_link_options(ydb-core-tx-datashard-ut_testload PRIVATE -lrt -ldl ) -target_sources(ydb-core-tx-datashard-ut_testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/datashard_ut_common.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/datashard_ut_testload.cpp +target_sources(ydb-core-load_test-ut_ycsb PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/load_test/ut_ycsb.cpp ) add_test( NAME - ydb-core-tx-datashard-ut_testload + ydb-core-load_test-ut_ycsb COMMAND - ydb-core-tx-datashard-ut_testload + ydb-core-load_test-ut_ycsb --print-before-suite --print-before-test --fork-tests --print-times --show-fails ) -vcs_info(ydb-core-tx-datashard-ut_testload) +vcs_info(ydb-core-load_test-ut_ycsb) diff --git a/ydb/core/tx/datashard/testload/CMakeLists.txt b/ydb/core/load_test/ut_ycsb/CMakeLists.txt index 3e0811fb22e..3e0811fb22e 100644 --- a/ydb/core/tx/datashard/testload/CMakeLists.txt +++ b/ydb/core/load_test/ut_ycsb/CMakeLists.txt diff --git a/ydb/core/tx/datashard/testload/actors.h b/ydb/core/load_test/ycsb/actors.h index 59a897a9ad0..542bb27eab5 100644 --- a/ydb/core/tx/datashard/testload/actors.h +++ b/ydb/core/load_test/ycsb/actors.h @@ -8,36 +8,36 @@ namespace NKikimr::NDataShardLoad { IActor *CreateUpsertBulkActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag); IActor *CreateLocalMkqlUpsertActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag); IActor *CreateKqpUpsertActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag); IActor *CreateProposeUpsertActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag); IActor *CreateReadIteratorActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TReadStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TReadStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag); diff --git a/ydb/core/tx/datashard/testload/bulk_mkql_upsert.cpp b/ydb/core/load_test/ycsb/bulk_mkql_upsert.cpp index 30a270ead5f..55c8ff3c904 100644 --- a/ydb/core/tx/datashard/testload/bulk_mkql_upsert.cpp +++ b/ydb/core/load_test/ycsb/bulk_mkql_upsert.cpp @@ -125,8 +125,8 @@ TRequestsVector GenerateRequests( } class TUpsertActor : public TActorBootstrapped<TUpsertActor> { - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart Config; - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard Target; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart Config; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard Target; const TActorId Parent; const ui64 Tag; const ERequestType RequestType; @@ -148,8 +148,8 @@ public: return NKikimrServices::TActivity::DS_LOAD_ACTOR; } - TUpsertActor(const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + TUpsertActor(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag, @@ -319,22 +319,22 @@ private: } // anonymous -IActor *CreateUpsertBulkActor(const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, +IActor *CreateUpsertBulkActor(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) { return new TUpsertActor(cmd, target, parent, std::move(counters), tag, ERequestType::UpsertBulk); } -IActor *CreateLocalMkqlUpsertActor(const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, +IActor *CreateLocalMkqlUpsertActor(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) { return new TUpsertActor(cmd, target, parent, std::move(counters), tag, ERequestType::UpsertLocalMkql); } -IActor *CreateProposeUpsertActor(const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, +IActor *CreateProposeUpsertActor(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) { Y_UNUSED(cmd); diff --git a/ydb/core/tx/datashard/testload/defs.h b/ydb/core/load_test/ycsb/defs.h index 56b6952963f..56b6952963f 100644 --- a/ydb/core/tx/datashard/testload/defs.h +++ b/ydb/core/load_test/ycsb/defs.h diff --git a/ydb/core/tx/datashard/testload/info_collector.cpp b/ydb/core/load_test/ycsb/info_collector.cpp index f82b9e4c34a..f82b9e4c34a 100644 --- a/ydb/core/tx/datashard/testload/info_collector.cpp +++ b/ydb/core/load_test/ycsb/info_collector.cpp diff --git a/ydb/core/tx/datashard/testload/info_collector.h b/ydb/core/load_test/ycsb/info_collector.h index 68cf9bafa33..68cf9bafa33 100644 --- a/ydb/core/tx/datashard/testload/info_collector.h +++ b/ydb/core/load_test/ycsb/info_collector.h diff --git a/ydb/core/tx/datashard/testload/kqp_upsert.cpp b/ydb/core/load_test/ycsb/kqp_upsert.cpp index b8ab23c5d2c..5fa970b116c 100644 --- a/ydb/core/tx/datashard/testload/kqp_upsert.cpp +++ b/ydb/core/load_test/ycsb/kqp_upsert.cpp @@ -76,8 +76,8 @@ TQueryInfo GenerateUpsert(size_t n, const TString& table) { // it's a partial copy-paste from TUpsertActor: logic slightly differs, so that // it seems better to have copy-paste rather if/else for different loads class TKqpUpsertActor : public TActorBootstrapped<TKqpUpsertActor> { - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart Config; - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard Target; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart Config; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard Target; const TActorId Parent; const ui64 Tag; const TString Database; @@ -95,8 +95,8 @@ class TKqpUpsertActor : public TActorBootstrapped<TKqpUpsertActor> { size_t Errors = 0; public: - TKqpUpsertActor(const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + TKqpUpsertActor(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag, @@ -246,8 +246,8 @@ private: // creates multiple TKqpUpsertActor for inflight > 1 and waits completion class TKqpUpsertActorMultiSession : public TActorBootstrapped<TKqpUpsertActorMultiSession> { - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart Config; - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard Target; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart Config; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard Target; const TActorId Parent; const ui64 Tag; TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters; @@ -266,8 +266,8 @@ class TKqpUpsertActorMultiSession : public TActorBootstrapped<TKqpUpsertActorMul size_t Errors = 0; public: - TKqpUpsertActorMultiSession(const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + TKqpUpsertActorMultiSession(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) @@ -428,8 +428,8 @@ private: } // anonymous IActor *CreateKqpUpsertActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) diff --git a/ydb/core/tx/datashard/testload/test_load_actor.cpp b/ydb/core/load_test/ycsb/test_load_actor.cpp index a3ee50ba1f2..8e005fdf60e 100644 --- a/ydb/core/tx/datashard/testload/test_load_actor.cpp +++ b/ydb/core/load_test/ycsb/test_load_actor.cpp @@ -5,6 +5,7 @@ #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/kqp/common/kqp.h> +#include <ydb/core/load_test/events.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/tx_proxy/proxy.h> @@ -15,20 +16,12 @@ // DataShard load is associated with full path to table: only one load per path can be started. // -// TLoadManager is a "service" actor which is used to start/stop special TLoad. It controls -// that there is only one TLoad per path. -// // TLoad is used to prepare database (drop, create, configure, etc) and then start load. -// -// Each TLoad actor has a tag TagStep 1000: i.e. 1000, 2000, 3000. Subactors created by -// TLoad must be within its Tag and next TagStep: i.e. 1001, 1002, etc namespace NKikimr::NDataShardLoad { namespace { -constexpr ui64 TagStep = 1000; - struct TFinishedTestInfo { ui64 Tag; TString ErrorReason; @@ -52,7 +45,7 @@ public: private: const TActorId Parent; const ui64 Tag; - NKikimrDataShardLoad::TEvTestLoadRequest Request; + NKikimrDataShardLoad::TEvYCSBTestLoadRequest Request; TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters; EState State = EState::Init; @@ -63,14 +56,11 @@ private: TString Session; TActorId HttpInfoCollector; - THashSet<TActorId> HttpInfoWaiters; + THashMap<TActorId, ui64> HttpInfoWaiters; // info about finished actors TVector<TFinishedTestInfo> FinishedTests; - enum class State { - }; - public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::DS_LOAD_ACTOR; @@ -78,7 +68,7 @@ public: TLoad(TActorId parent, ui64 tag, - NKikimrDataShardLoad::TEvTestLoadRequest&& request, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest& request, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) : Parent(parent) , Tag(tag) @@ -319,21 +309,21 @@ public: return; } - NKikimrDataShardLoad::TEvTestLoadRequest::TUpdateStart cmd; + NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TUpdateStart cmd; switch (Request.Command_case()) { - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertBulkStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertBulkStart: cmd = Request.GetUpsertBulkStart(); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertLocalMkqlStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertLocalMkqlStart: cmd = Request.GetUpsertLocalMkqlStart(); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertKqpStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertKqpStart: cmd = Request.GetUpsertKqpStart(); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertProposeStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertProposeStart: cmd = Request.GetUpsertProposeStart(); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kReadIteratorStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kReadIteratorStart: cmd.SetRowCount(Request.GetReadIteratorStart().GetRowCount()); break; default: @@ -366,7 +356,7 @@ public: auto counters = GetServiceCounters(Counters, "load_actor"); switch (Request.Command_case()) { - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertBulkStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertBulkStart: actor.reset(CreateUpsertBulkActor( Request.GetUpsertBulkStart(), Request.GetTargetShard(), @@ -374,7 +364,7 @@ public: counters, tag)); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertLocalMkqlStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertLocalMkqlStart: actor.reset(CreateLocalMkqlUpsertActor( Request.GetUpsertLocalMkqlStart(), Request.GetTargetShard(), @@ -382,7 +372,7 @@ public: counters, tag)); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertKqpStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertKqpStart: actor.reset(CreateKqpUpsertActor( Request.GetUpsertKqpStart(), Request.GetTargetShard(), @@ -390,7 +380,7 @@ public: counters, tag)); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kUpsertProposeStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kUpsertProposeStart: actor.reset(CreateProposeUpsertActor( Request.GetUpsertProposeStart(), Request.GetTargetShard(), @@ -398,7 +388,7 @@ public: counters, tag)); break; - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kReadIteratorStart: + case NKikimrDataShardLoad::TEvYCSBTestLoadRequest::CommandCase::kReadIteratorStart: actor.reset(CreateReadIteratorActor( Request.GetReadIteratorStart(), Request.GetTargetShard(), @@ -450,79 +440,96 @@ public: void Finish(const TActorContext& ctx) { auto endTs = TAppData::TimeProvider->Now(); - - auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadFinished>(Tag); - auto& report = *response->Record.MutableReport(); - report.SetTag(Tag); - report.SetDurationMs((endTs - StartTs).MilliSeconds()); + auto duration = endTs - StartTs; ui64 oks = 0; ui64 errors = 0; ui64 subtestCount = 0; - TStringStream ss; for (const auto& test: FinishedTests) { oks += test.Report.GetOperationsOK(); errors += test.Report.GetOperationsError(); subtestCount += test.Report.GetSubtestCount(); - if (test.Report.HasInfo()) - ss << test.Report.GetInfo() << Endl; } - report.SetOperationsOK(oks); - report.SetOperationsError(errors); - report.SetSubtestCount(subtestCount); + TIntrusivePtr<TEvLoad::TLoadReport> report(new TEvLoad::TLoadReport()); + report->Duration = duration; + + NJson::TJsonValue value; + value["duration_s"] = duration.Seconds(); + value["oks"] = oks; + value["errors"] = errors; + value["subtests"] = subtestCount; - ctx.Send(Parent, response.release()); + auto finishEv = std::make_unique<TEvLoad::TEvLoadTestFinished>(Tag, report); + finishEv->JsonResult = std::move(value); + ctx.Send(Parent, finishEv.release()); Die(ctx); } - void Handle(TEvDataShardLoad::TEvTestLoadInfoRequest::TPtr& ev, const TActorContext& ctx) { - if (LoadActors.empty()) { - TStringStream ss; - ss << "TLoad# " << Tag << " started on " << StartTs - << " with no subactors, finished# " << FinishedTests.size() - << ", in state# " << State; - - auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadInfoResponse>(); - auto* info = response->Record.AddReports(); - info->SetTag(Tag); - info->SetInfo(ss.Str()); - ctx.Send(ev->Sender, response.release()); + void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { + HttpInfoWaiters[ev->Sender] = ev->Get()->SubRequestId; + if (HttpInfoCollector) { return; } - HttpInfoWaiters.insert(ev->Sender); - if (HttpInfoCollector) + if (LoadActors.empty()) { + auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadInfoResponse>(); + ctx.Send(ctx.SelfID, response.release()); return; + } TVector<TActorId> actors; actors.reserve(LoadActors.size()); - for (const auto& actorId : LoadActors) { - actors.push_back(actorId); + for (const auto& actor : LoadActors) { + actors.push_back(actor); } HttpInfoCollector = ctx.Register(CreateInfoCollector(SelfId(), std::move(actors))); } void Handle(TEvDataShardLoad::TEvTestLoadInfoResponse::TPtr& ev, const TActorContext& ctx) { - TStringStream ss; - ss << "TLoad# " << Tag << " started on " << StartTs - << " with subactors active# " << LoadActors.size() - << ", finished# " << FinishedTests.size() - << ", subactors infos: "; - + // aggregate total info + ui64 oks = 0; + ui64 errors = 0; + ui64 subtestCount = 0; for (auto& info: ev->Get()->Record.GetReports()) { - ss << "{ tag: " << info.GetTag() << ", info: " << info.GetInfo() << " }"; + oks += info.GetOperationsOK(); + errors += info.GetOperationsError(); + subtestCount += info.GetSubtestCount(); } - NKikimrDataShardLoad::TEvTestLoadInfoResponse record; - auto* report = record.AddReports(); - report->SetTag(Tag); - report->SetInfo(ss.Str()); + for (const auto& result: FinishedTests) { + oks += result.Report.GetOperationsOK(); + errors += result.Report.GetOperationsError(); + subtestCount += result.Report.GetSubtestCount(); + } - for (const auto& actorId: HttpInfoWaiters) { - auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadInfoResponse>(); - response->Record = record; - ctx.Send(actorId, response.release()); +#define PARAM(NAME, VALUE) \ + TABLER() { \ + TABLED() { str << NAME; } \ + TABLED() { str << VALUE; } \ + } + TStringStream str; + HTML(str) { + TABLE_CLASS("table table-condensed") { + TABLEHEAD() { + TABLER() { + TABLEH() { str << "Parameter"; } + TABLEH() { str << "Value"; } + } + } + TABLEBODY() { + PARAM("Elapsed time", (TAppData::TimeProvider->Now() - StartTs).Seconds() << "s"); + PARAM("OKs", oks); + PARAM("Errors", errors); + PARAM("Running subactors", LoadActors.size()) + PARAM("Finished subactors", FinishedTests.size()) + } + } + } +#undef PARAM + + for (const auto& it: HttpInfoWaiters) { + ctx.Send(it.first, new NMon::TEvHttpInfoRes(str.Str(), it.second)); } HttpInfoWaiters.clear(); @@ -555,236 +562,26 @@ public: STRICT_STFUNC(StateFunc, HFunc(TEvDataShardLoad::TEvTestLoadFinished, Handle) - HFunc(TEvDataShardLoad::TEvTestLoadInfoRequest, Handle) HFunc(TEvDataShardLoad::TEvTestLoadInfoResponse, Handle) HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle) HFunc(NKqp::TEvKqp::TEvCreateSessionResponse, Handle) HFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle) + HFunc(NMon::TEvHttpInfo, Handle) CFunc(TEvents::TSystem::PoisonPill, HandlePoison) ) }; // TLoadManager -class TLoadManager : public TActorBootstrapped<TLoadManager> { - struct TRunningActorInfo { - TActorId ActorId; - TActorId Parent; // if set we notify parent when actor finishes - - explicit TRunningActorInfo(const TActorId& actorId, const TActorId& parent = {}) - : ActorId(actorId) - , Parent(parent) - { - } - }; - - // info about finished actors - TVector<TFinishedTestInfo> FinishedTests; - - // currently running load actors - TMap<ui64, TRunningActorInfo> LoadActors; - - ui64 LastTag = 0; // tags start from TagStep - - THashMap<TActorId, ui64> HttpInfoWaiters; - TActorId HttpInfoCollector; - - TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::DS_LOAD_ACTOR; - } - - TLoadManager(const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) - : Counters(counters) - {} - - void Bootstrap(const TActorContext& /*ctx*/) { - Become(&TLoadManager::StateFunc); - } - - void Handle(TEvDataShardLoad::TEvTestLoadRequest::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - ui32 status = NMsgBusProxy::MSTATUS_OK; - TString error; - ui64 tag = 0; - try { - tag = ProcessCmd(ev, ctx); - } catch (const TLoadManagerException& ex) { - LOG_ERROR_S(ctx, NKikimrServices::DS_LOAD_TEST, "Exception while creating load actor, what# " - << ex.what()); - status = NMsgBusProxy::MSTATUS_ERROR; - error = ex.what(); - } - auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadResponse>(); - response->Record.SetStatus(status); - if (error) { - response->Record.SetErrorReason(error); - } - if (record.HasCookie()) { - response->Record.SetCookie(record.GetCookie()); - } - if (tag) { - response->Record.SetTag(tag); - } - ctx.Send(ev->Sender, response.release()); - } - - ui64 GetTag() { - LastTag += TagStep; - - // just sanity check - if (LoadActors.contains(LastTag)) { - ythrow TLoadManagerException() << Sprintf("duplicate load actor with Tag# %" PRIu64, LastTag); - } - - return LastTag; - } - - ui64 ProcessCmd(TEvDataShardLoad::TEvTestLoadRequest::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - switch (record.Command_case()) { - case NKikimrDataShardLoad::TEvTestLoadRequest::CommandCase::kLoadStop: { - const auto& cmd = record.GetLoadStop(); - if (cmd.HasRemoveAllTags() && cmd.GetRemoveAllTags()) { - LOG_DEBUG_S(ctx, NKikimrServices::DS_LOAD_TEST, "Delete all running load actors"); - for (auto& actorPair : LoadActors) { - ctx.Send(actorPair.second.ActorId, new TEvents::TEvPoisonPill); - } - LoadActors.clear(); - } else { - if (!cmd.HasTag()) { - ythrow TLoadManagerException() << "Either RemoveAllTags or Tag must present"; - } - const ui64 tag = cmd.GetTag(); - auto it = LoadActors.find(tag); - if (it == LoadActors.end()) { - ythrow TLoadManagerException() - << Sprintf("load actor with Tag# %" PRIu64 " not found", tag); - } - LOG_DEBUG_S(ctx, NKikimrServices::DS_LOAD_TEST, "Delete running load actor# " << tag); - ctx.Send(it->second.ActorId, new TEvents::TEvPoisonPill); - LoadActors.erase(it); - } - - return 0; - } - default: { - const ui64 tag = GetTag(); - bool notifyParent = record.GetNotifyWhenFinished(); - TRunningActorInfo actorInfo(ctx.Register(new TLoad(SelfId(), tag, std::move(record), Counters))); - if (notifyParent) - actorInfo.Parent = ev->Sender; - LoadActors.emplace(tag, std::move(actorInfo)); - return tag; - } - } - } - - void Handle(TEvDataShardLoad::TEvTestLoadFinished::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - auto it = LoadActors.find(record.GetTag()); - - Y_VERIFY(it != LoadActors.end(), "%s", (TStringBuilder() << "failed to find actor with tag# " << record.GetTag() - << ", TEvTestLoadFinished from actor# " << ev->Sender).c_str()); - LOG_DEBUG_S(ctx, NKikimrServices::DS_LOAD_TEST, "Load actor# " << ev->Sender - << " with tag# " << record.GetTag() << " finished"); - - if (it->second.Parent) { - auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadFinished>(record.GetTag()); - response->Record = record; - ctx.Send(it->second.Parent, response.release()); - } - - LoadActors.erase(it); - FinishedTests.push_back( - {record.GetTag(), record.GetErrorReason(), TAppData::TimeProvider->Now(), record.GetReport()}); - } - - void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { - HttpInfoWaiters[ev->Sender] = ev->Get()->SubRequestId; - if (HttpInfoCollector) { - return; - } - - if (LoadActors.empty()) { - auto response = std::make_unique<TEvDataShardLoad::TEvTestLoadInfoResponse>(); - ctx.Send(ctx.SelfID, response.release()); - return; - } - - TVector<TActorId> actors; - actors.reserve(LoadActors.size()); - for (const auto& kv : LoadActors) { - actors.push_back(kv.second.ActorId); - } - HttpInfoCollector = ctx.Register(CreateInfoCollector(SelfId(), std::move(actors))); - } - - void Handle(TEvDataShardLoad::TEvTestLoadInfoResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - - TStringStream str; - HTML(str) { - DIV_CLASS("panel panel-info") { - DIV_CLASS("panel-heading") { - str << "State"; - } - DIV_CLASS("panel-body") { - str << "Running# " << LoadActors.size() << ", Finished# " << FinishedTests.size(); - } - } - - for (const auto& info: record.GetReports()) { - DIV_CLASS("panel panel-info") { - DIV_CLASS("panel-heading") { - str << "Tag# " << info.GetTag(); - } - DIV_CLASS("panel-body") { - str << info.GetInfo(); - } - } - } - - COLLAPSED_BUTTON_CONTENT("finished_tests_info", "Finished tests") { - for (const auto& req : FinishedTests) { - DIV_CLASS("panel panel-info") { - DIV_CLASS("panel-heading") { - str << "Tag# " << req.Tag; - } - DIV_CLASS("panel-body") { - str << "<p>"; - str << "Report# " << req.Report << "<br/>"; - str << "Finish reason# " << req.ErrorReason << "<br/>"; - str << "Finish time# " << req.FinishTime << "<br/>"; - str << "</p>"; - } - } - } - } - } - - for (const auto& it: HttpInfoWaiters) { - ctx.Send(it.first, new NMon::TEvHttpInfoRes(str.Str(), it.second)); - } - - HttpInfoWaiters.clear(); - HttpInfoCollector = {}; - } - - STRICT_STFUNC(StateFunc, - HFunc(TEvDataShardLoad::TEvTestLoadRequest, Handle) - HFunc(TEvDataShardLoad::TEvTestLoadFinished, Handle) - HFunc(TEvDataShardLoad::TEvTestLoadInfoResponse, Handle) - HFunc(NMon::TEvHttpInfo, Handle) - ) -}; - } // anonymous -IActor *CreateTestLoadActor(const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) { - return new TLoadManager(counters); +IActor *CreateTestLoadActor( + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest& request, + TActorId parent, + const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters, + ui64 tag) +{ + return new TLoad(parent, tag, request, counters); } } // NKikimr::NDataShardLoad diff --git a/ydb/core/tx/datashard/testload/test_load_actor.h b/ydb/core/load_test/ycsb/test_load_actor.h index f6ac06af3e3..ff8e95e59fe 100644 --- a/ydb/core/tx/datashard/testload/test_load_actor.h +++ b/ydb/core/load_test/ycsb/test_load_actor.h @@ -20,12 +20,12 @@ struct TEvDataShardLoad { EvTestLoadInfoResponse, }; - struct TEvTestLoadRequest - : public TEventPB<TEvTestLoadRequest, - NKikimrDataShardLoad::TEvTestLoadRequest, + struct TEvYCSBTestLoadRequest + : public TEventPB<TEvYCSBTestLoadRequest, + NKikimrDataShardLoad::TEvYCSBTestLoadRequest, EvTestLoadRequest> { - TEvTestLoadRequest() = default; + TEvYCSBTestLoadRequest() = default; }; struct TEvTestLoadResponse @@ -89,7 +89,11 @@ struct TEvDataShardLoad { namespace NDataShardLoad { -IActor *CreateTestLoadActor(const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters); +IActor *CreateTestLoadActor( + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest& request, + TActorId parent, + const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters, + ui64 tag); } // NDataShardLoad } // NKikimr diff --git a/ydb/core/tx/datashard/testload/test_load_read_iterator.cpp b/ydb/core/load_test/ycsb/test_load_read_iterator.cpp index 46da2960aaf..fbb2906f485 100644 --- a/ydb/core/tx/datashard/testload/test_load_read_iterator.cpp +++ b/ydb/core/load_test/ycsb/test_load_read_iterator.cpp @@ -403,8 +403,8 @@ enum class EState { }; class TReadIteratorLoadScenario : public TActorBootstrapped<TReadIteratorLoadScenario> { - const NKikimrDataShardLoad::TEvTestLoadRequest::TReadStart Config; - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard Target; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TReadStart Config; + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard Target; const TActorId Parent; TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters; const ui64 Tag; @@ -448,8 +448,8 @@ class TReadIteratorLoadScenario : public TActorBootstrapped<TReadIteratorLoadSce ui64 ReadCount = 0; public: - TReadIteratorLoadScenario(const NKikimrDataShardLoad::TEvTestLoadRequest::TReadStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + TReadIteratorLoadScenario(const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TReadStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) @@ -808,8 +808,8 @@ private: } // anonymous IActor *CreateReadIteratorActor( - const NKikimrDataShardLoad::TEvTestLoadRequest::TReadStart& cmd, - const NKikimrDataShardLoad::TEvTestLoadRequest::TTargetShard& target, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TReadStart& cmd, + const NKikimrDataShardLoad::TEvYCSBTestLoadRequest::TTargetShard& target, const TActorId& parent, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters, ui64 tag) { diff --git a/ydb/core/protos/datashard_load.proto b/ydb/core/protos/datashard_load.proto index a0f4f0ca222..1e79fe889ba 100644 --- a/ydb/core/protos/datashard_load.proto +++ b/ydb/core/protos/datashard_load.proto @@ -5,7 +5,7 @@ import "ydb/core/protos/tx_datashard.proto"; package NKikimrDataShardLoad; option java_package = "ru.yandex.kikimr.proto"; -message TEvTestLoadRequest { +message TEvYCSBTestLoadRequest { message TLoadStop { optional uint64 Tag = 1; optional bool RemoveAllTags = 2; @@ -65,7 +65,7 @@ message TEvTestLoadRequest { optional bool SkipWarmup = 8; } - optional uint64 Cookie = 1; + optional uint64 Tag = 1; // normally should be used by ut only optional bool NotifyWhenFinished = 2; diff --git a/ydb/core/protos/load_test.proto b/ydb/core/protos/load_test.proto index 0ab718b3f20..f418786feac 100644 --- a/ydb/core/protos/load_test.proto +++ b/ydb/core/protos/load_test.proto @@ -1,5 +1,6 @@ import "ydb/core/protos/blobstorage.proto"; import "ydb/core/protos/blobstorage_disk.proto"; +import "ydb/core/protos/datashard_load.proto"; package NKikimr; option java_package = "ru.yandex.kikimr.proto"; @@ -240,6 +241,7 @@ message TEvLoadTestRequest { TKeyValueLoad KeyValueLoad = 8; TKqpLoad KqpLoad = 9; TMemoryLoad MemoryLoad = 10; + NKikimrDataShardLoad.TEvYCSBTestLoadRequest YCSBLoad = 11; } } diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index 81e5d01d7dc..5fb2d8eb838 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -467,7 +467,7 @@ message TSchemeOperationStatus { message TDsTestLoadRequest { optional uint32 NodeId = 1; - optional NKikimrDataShardLoad.TEvTestLoadRequest Event = 2; + optional NKikimrDataShardLoad.TEvYCSBTestLoadRequest Event = 2; }; message TDsTestLoadResponse { diff --git a/ydb/core/tx/datashard/CMakeLists.darwin.txt b/ydb/core/tx/datashard/CMakeLists.darwin.txt index 474414f0000..82728a254db 100644 --- a/ydb/core/tx/datashard/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/CMakeLists.darwin.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(testload) add_subdirectory(ut_background_compaction) add_subdirectory(ut_build_index) add_subdirectory(ut_change_collector) @@ -33,7 +32,6 @@ add_subdirectory(ut_replication) add_subdirectory(ut_rs) add_subdirectory(ut_snapshot) add_subdirectory(ut_stats) -add_subdirectory(ut_testload) add_subdirectory(ut_upload_rows) add_subdirectory(ut_volatile) @@ -68,7 +66,6 @@ target_link_libraries(core-tx-datashard PUBLIC ydb-core-protos ydb-core-tablet ydb-core-tablet_flat - tx-datashard-testload tx-long_tx_service-public ydb-core-util ydb-core-wrappers @@ -322,7 +319,6 @@ target_link_libraries(core-tx-datashard.global PUBLIC ydb-core-protos ydb-core-tablet ydb-core-tablet_flat - tx-datashard-testload tx-long_tx_service-public ydb-core-util ydb-core-wrappers diff --git a/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt index 8a3825a0dc1..3dbda17f127 100644 --- a/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(testload) add_subdirectory(ut_background_compaction) add_subdirectory(ut_build_index) add_subdirectory(ut_change_collector) @@ -33,7 +32,6 @@ add_subdirectory(ut_replication) add_subdirectory(ut_rs) add_subdirectory(ut_snapshot) add_subdirectory(ut_stats) -add_subdirectory(ut_testload) add_subdirectory(ut_upload_rows) add_subdirectory(ut_volatile) @@ -69,7 +67,6 @@ target_link_libraries(core-tx-datashard PUBLIC ydb-core-protos ydb-core-tablet ydb-core-tablet_flat - tx-datashard-testload tx-long_tx_service-public ydb-core-util ydb-core-wrappers @@ -324,7 +321,6 @@ target_link_libraries(core-tx-datashard.global PUBLIC ydb-core-protos ydb-core-tablet ydb-core-tablet_flat - tx-datashard-testload tx-long_tx_service-public ydb-core-util ydb-core-wrappers diff --git a/ydb/core/tx/datashard/CMakeLists.linux.txt b/ydb/core/tx/datashard/CMakeLists.linux.txt index 8a3825a0dc1..3dbda17f127 100644 --- a/ydb/core/tx/datashard/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/CMakeLists.linux.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(testload) add_subdirectory(ut_background_compaction) add_subdirectory(ut_build_index) add_subdirectory(ut_change_collector) @@ -33,7 +32,6 @@ add_subdirectory(ut_replication) add_subdirectory(ut_rs) add_subdirectory(ut_snapshot) add_subdirectory(ut_stats) -add_subdirectory(ut_testload) add_subdirectory(ut_upload_rows) add_subdirectory(ut_volatile) @@ -69,7 +67,6 @@ target_link_libraries(core-tx-datashard PUBLIC ydb-core-protos ydb-core-tablet ydb-core-tablet_flat - tx-datashard-testload tx-long_tx_service-public ydb-core-util ydb-core-wrappers @@ -324,7 +321,6 @@ target_link_libraries(core-tx-datashard.global PUBLIC ydb-core-protos ydb-core-tablet ydb-core-tablet_flat - tx-datashard-testload tx-long_tx_service-public ydb-core-util ydb-core-wrappers diff --git a/ydb/core/tx/datashard/testload/CMakeLists.darwin.txt b/ydb/core/tx/datashard/testload/CMakeLists.darwin.txt deleted file mode 100644 index e40eec92328..00000000000 --- a/ydb/core/tx/datashard/testload/CMakeLists.darwin.txt +++ /dev/null @@ -1,30 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(tx-datashard-testload) -target_link_libraries(tx-datashard-testload PUBLIC - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf - monlib-dynamic_counters-percentile - monlib-service-pages - ydb-core-base - ydb-core-control - ydb-core-keyvalue - public-lib-base - public-lib-operation_id - cpp-client-ydb_proto -) -target_sources(tx-datashard-testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/bulk_mkql_upsert.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/info_collector.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/kqp_upsert.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/test_load_actor.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/test_load_read_iterator.cpp -) diff --git a/ydb/core/tx/datashard/testload/CMakeLists.linux-aarch64.txt b/ydb/core/tx/datashard/testload/CMakeLists.linux-aarch64.txt deleted file mode 100644 index c7399345e5a..00000000000 --- a/ydb/core/tx/datashard/testload/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,31 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(tx-datashard-testload) -target_link_libraries(tx-datashard-testload PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf - monlib-dynamic_counters-percentile - monlib-service-pages - ydb-core-base - ydb-core-control - ydb-core-keyvalue - public-lib-base - public-lib-operation_id - cpp-client-ydb_proto -) -target_sources(tx-datashard-testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/bulk_mkql_upsert.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/info_collector.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/kqp_upsert.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/test_load_actor.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/test_load_read_iterator.cpp -) diff --git a/ydb/core/tx/datashard/testload/CMakeLists.linux.txt b/ydb/core/tx/datashard/testload/CMakeLists.linux.txt deleted file mode 100644 index c7399345e5a..00000000000 --- a/ydb/core/tx/datashard/testload/CMakeLists.linux.txt +++ /dev/null @@ -1,31 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(tx-datashard-testload) -target_link_libraries(tx-datashard-testload PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf - monlib-dynamic_counters-percentile - monlib-service-pages - ydb-core-base - ydb-core-control - ydb-core-keyvalue - public-lib-base - public-lib-operation_id - cpp-client-ydb_proto -) -target_sources(tx-datashard-testload PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/bulk_mkql_upsert.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/info_collector.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/kqp_upsert.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/test_load_actor.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/datashard/testload/test_load_read_iterator.cpp -) diff --git a/ydb/core/tx/datashard/ut_testload/CMakeLists.txt b/ydb/core/tx/datashard/ut_testload/CMakeLists.txt deleted file mode 100644 index 3e0811fb22e..00000000000 --- a/ydb/core/tx/datashard/ut_testload/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) - include(CMakeLists.linux-aarch64.txt) -elseif (APPLE) - include(CMakeLists.darwin.txt) -elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) - include(CMakeLists.linux.txt) -endif() diff --git a/ydb/public/lib/base/msgbus.h b/ydb/public/lib/base/msgbus.h index a895b22e331..3fa99faa95d 100644 --- a/ydb/public/lib/base/msgbus.h +++ b/ydb/public/lib/base/msgbus.h @@ -89,8 +89,8 @@ enum { MTYPE_CLIENT_TENANT_SLOT_BROKER_REQUEST = 10479, MTYPE_CLIENT_TENANT_SLOT_BROKER_RESPONSE = 10480, MTYPE_CLIENT_TEST_SHARD_CONTROL = 10481, - MTYPE_CLIENT_DS_LOAD_REQUEST = 10482, - MTYPE_CLIENT_DS_LOAD_RESPONSE = 10483, + MTYPE_CLIENT_DS_LOAD_REQUEST = 10482, // deprecated + MTYPE_CLIENT_DS_LOAD_RESPONSE = 10483, // deprecated MTYPE_CLIENT_LOGIN_REQUEST = 10484, }; @@ -127,8 +127,6 @@ struct TBusSchemeOperationStatus : TBusMessage<TBusSchemeOperationStatus, NKikim struct TBusSchemeDescribe : TBusMessage<TBusSchemeDescribe, NKikimrClient::TSchemeDescribe, MTYPE_CLIENT_FLAT_DESCRIBE_REQUEST> {}; struct TBusOldFlatDescribeRequest : TBusMessage<TBusOldFlatDescribeRequest, NKikimrClient::TSchemeDescribe, MTYPE_CLIENT_OLD_FLAT_DESCRIBE_REQUEST> {}; struct TBusOldFlatDescribeResponse : TBusMessage<TBusOldFlatDescribeResponse, NKikimrClient::TFlatDescribeResponse, MTYPE_CLIENT_OLD_FLAT_DESCRIBE_RESPONSE> {}; -struct TBusDsTestLoadRequest : TBusMessage<TBusDsTestLoadRequest, NKikimrClient::TDsTestLoadRequest, MTYPE_CLIENT_DS_LOAD_REQUEST> {}; -struct TBusDsTestLoadResponse : TBusMessage<TBusDsTestLoadResponse, NKikimrClient::TDsTestLoadResponse, MTYPE_CLIENT_DS_LOAD_RESPONSE> {}; struct TBusBsTestLoadRequest : TBusMessage<TBusBsTestLoadRequest, NKikimrClient::TBsTestLoadRequest, MTYPE_CLIENT_LOAD_REQUEST> {}; struct TBusBsTestLoadResponse : TBusMessage<TBusBsTestLoadResponse, NKikimrClient::TBsTestLoadResponse, MTYPE_CLIENT_LOAD_RESPONSE> {}; struct TBusBsGetRequest : TBusMessage<TBusBsGetRequest, NKikimrClient::TBsGetRequest, MTYPE_CLIENT_GET_REQUEST> {}; @@ -225,8 +223,6 @@ public: RegisterType(new TBusSchemeDescribe); RegisterType(new TBusOldFlatDescribeRequest); RegisterType(new TBusOldFlatDescribeResponse); - RegisterType(new TBusDsTestLoadRequest); - RegisterType(new TBusDsTestLoadResponse); RegisterType(new TBusBsTestLoadRequest); RegisterType(new TBusBsTestLoadResponse); RegisterType(new TBusBsGetRequest); diff --git a/ydb/public/lib/deprecated/kicli/kikimr.cpp b/ydb/public/lib/deprecated/kicli/kikimr.cpp index 364c89ac876..028275f6776 100644 --- a/ydb/public/lib/deprecated/kicli/kikimr.cpp +++ b/ydb/public/lib/deprecated/kicli/kikimr.cpp @@ -259,8 +259,6 @@ public: return ExecuteGRpcRequest<NMsgBusProxy::TBusTabletKillRequest>(&NGRpcProxy::TGRpcClient::TabletKillRequest, promise, request); case NMsgBusProxy::MTYPE_CLIENT_TABLET_STATE_REQUEST: return ExecuteGRpcRequest<NMsgBusProxy::TBusTabletStateRequest>(&NGRpcProxy::TGRpcClient::TabletStateRequest, promise, request); - case NMsgBusProxy::MTYPE_CLIENT_DS_LOAD_REQUEST: - return ExecuteGRpcRequest<NMsgBusProxy::TBusDsTestLoadRequest>(&NGRpcProxy::TGRpcClient::DataShardLoadRequest, promise, request); case NMsgBusProxy::MTYPE_CLIENT_LOAD_REQUEST: return ExecuteGRpcRequest<NMsgBusProxy::TBusBsTestLoadRequest>(&NGRpcProxy::TGRpcClient::BlobStorageLoadRequest, promise, request); case NMsgBusProxy::MTYPE_CLIENT_GET_REQUEST: diff --git a/ydb/public/lib/deprecated/kicli/result.cpp b/ydb/public/lib/deprecated/kicli/result.cpp index eeb6dded935..0f86ba4340a 100644 --- a/ydb/public/lib/deprecated/kicli/result.cpp +++ b/ydb/public/lib/deprecated/kicli/result.cpp @@ -36,11 +36,6 @@ template <> const NKikimrClient::TBsTestLoadResponse& TResult::GetResult<NKikimr return static_cast<NMsgBusProxy::TBusBsTestLoadResponse*>(Reply.Get())->Record; } -template <> const NKikimrClient::TDsTestLoadResponse& TResult::GetResult<NKikimrClient::TDsTestLoadResponse>() const { - Y_VERIFY(GetType() == NMsgBusProxy::MTYPE_CLIENT_DS_LOAD_RESPONSE, "Unexpected response type: %d", GetType()); - return static_cast<NMsgBusProxy::TBusDsTestLoadResponse*>(Reply.Get())->Record; -} - NMsgBusProxy::EResponseStatus TResult::GetStatus() const { if (TransportStatus != NBus::MESSAGE_OK) { switch (TransportStatus) { |