diff options
| author | gleb-kov <[email protected]> | 2022-02-10 16:46:22 +0300 |
|---|---|---|
| committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:46:22 +0300 |
| commit | 7b1cfa32681104c8468c5824c79fd80d9a88a579 (patch) | |
| tree | 499e0a6e0f35aba718ac262605efb4342996bdca | |
| parent | 89973d10cfeccd97d2f25449a523c93940565edf (diff) | |
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
115 files changed, 1927 insertions, 1927 deletions
diff --git a/build/stdafx.hpp b/build/stdafx.hpp index 98b15adda05..eff2240a99c 100644 --- a/build/stdafx.hpp +++ b/build/stdafx.hpp @@ -135,7 +135,7 @@ #include <util/system/shmat.h> #include <util/system/rusage.h> #include <util/system/yassert.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/system/atomic.h> #include <util/system/yield.h> #include <util/system/filemap.h> @@ -163,7 +163,7 @@ #include <util/system/datetime.h> #include <util/system/utime.h> #include <util/system/mlock.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/file.h> #include <util/system/atexit.h> #include <util/system/context_x86.h> diff --git a/build/ya.conf.json b/build/ya.conf.json index 5f7cc875d61..5809b385b48 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -5767,9 +5767,9 @@ "clang-tidy": { "formula": { "sandbox_id": [ - 1073581733, - 1073610805, - 1073627148 + 1073581733, + 1073610805, + 1073627148 ], "match": "CLANG_TIDY" }, diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 081833998b8..1b72a00ff18 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -12,7 +12,7 @@ SANDBOX_FAKEID=${FAKEID}.7600000 CPP_FAKEID=9107927 GO_FAKEID=9056219 ANDROID_FAKEID=8821472 -CLANG_TIDY_FAKEID=8625699 +CLANG_TIDY_FAKEID=8625699 CURDIR=. MODDIR=. diff --git a/contrib/restricted/boost/boost/iterator/counting_iterator.hpp b/contrib/restricted/boost/boost/iterator/counting_iterator.hpp index 80108f15061..f994bab7322 100644 --- a/contrib/restricted/boost/boost/iterator/counting_iterator.hpp +++ b/contrib/restricted/boost/boost/iterator/counting_iterator.hpp @@ -170,8 +170,8 @@ class counting_iterator { } - counting_iterator& operator=(counting_iterator const&) = default; - + counting_iterator& operator=(counting_iterator const&) = default; + # if 0 template<class OtherIncrementable> counting_iterator( diff --git a/contrib/restricted/boost/boost/multiprecision/detail/number_base.hpp b/contrib/restricted/boost/boost/multiprecision/detail/number_base.hpp index 24e0299b153..75e687dbdf4 100644 --- a/contrib/restricted/boost/boost/multiprecision/detail/number_base.hpp +++ b/contrib/restricted/boost/boost/multiprecision/detail/number_base.hpp @@ -408,7 +408,7 @@ struct expression<tag, Arg1, void, void, void> typedef tag tag_type; explicit expression(const Arg1& a) : arg(a) {} - expression(const expression& e) : arg(e.arg) {} + expression(const expression& e) : arg(e.arg) {} left_type left()const { return left_type(arg); } @@ -480,7 +480,7 @@ struct expression<terminal, Arg1, void, void, void> typedef terminal tag_type; explicit expression(const Arg1& a) : arg(a) {} - expression(const expression& e) : arg(e.arg) {} + expression(const expression& e) : arg(e.arg) {} const Arg1& value()const BOOST_NOEXCEPT { return arg; } @@ -554,7 +554,7 @@ struct expression<tag, Arg1, Arg2, void, void> typedef tag tag_type; expression(const Arg1& a1, const Arg2& a2) : arg1(a1), arg2(a2) {} - expression(const expression& e) : arg1(e.arg1), arg2(e.arg2) {} + expression(const expression& e) : arg1(e.arg1), arg2(e.arg2) {} left_type left()const { return left_type(arg1); } right_type right()const { return right_type(arg2); } @@ -638,7 +638,7 @@ struct expression<tag, Arg1, Arg2, Arg3, void> typedef tag tag_type; expression(const Arg1& a1, const Arg2& a2, const Arg3& a3) : arg1(a1), arg2(a2), arg3(a3) {} - expression(const expression& e) : arg1(e.arg1), arg2(e.arg2), arg3(e.arg3) {} + expression(const expression& e) : arg1(e.arg1), arg2(e.arg2), arg3(e.arg3) {} left_type left()const { return left_type(arg1); } middle_type middle()const { return middle_type(arg2); } @@ -731,7 +731,7 @@ struct expression typedef tag tag_type; expression(const Arg1& a1, const Arg2& a2, const Arg3& a3, const Arg4& a4) : arg1(a1), arg2(a2), arg3(a3), arg4(a4) {} - expression(const expression& e) : arg1(e.arg1), arg2(e.arg2), arg3(e.arg3), arg4(e.arg4) {} + expression(const expression& e) : arg1(e.arg1), arg2(e.arg2), arg3(e.arg3), arg4(e.arg4) {} left_type left()const { return left_type(arg1); } left_middle_type left_middle()const { return left_middle_type(arg2); } diff --git a/contrib/restricted/boost/boost/process/detail/posix/sigchld_service.hpp b/contrib/restricted/boost/boost/process/detail/posix/sigchld_service.hpp index 437be438bec..0112fa54279 100644 --- a/contrib/restricted/boost/boost/process/detail/posix/sigchld_service.hpp +++ b/contrib/restricted/boost/boost/process/detail/posix/sigchld_service.hpp @@ -104,7 +104,7 @@ void sigchld_service::_handle_signal(const boost::system::error_code & ec) _signal_set.async_wait( [this](const boost::system::error_code & ec, int) { - _strand.post([]{}); + _strand.post([]{}); this->_handle_signal(ec); }); } diff --git a/library/cpp/actors/core/actor_coroutine.cpp b/library/cpp/actors/core/actor_coroutine.cpp index 0ab4d2b24de..3fd39f74318 100644 --- a/library/cpp/actors/core/actor_coroutine.cpp +++ b/library/cpp/actors/core/actor_coroutine.cpp @@ -2,7 +2,7 @@ #include "executor_thread.h" #include <util/system/sanitizers.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NActors { static constexpr size_t StackOverflowGap = 4096; @@ -143,7 +143,7 @@ namespace NActors { Y_FAIL("unhandled TDtorException"); } } catch (const std::exception& ex) { - Y_FAIL("unhandled exception of type %s", TypeName(ex).data()); + Y_FAIL("unhandled exception of type %s", TypeName(ex).data()); } catch (...) { Y_FAIL("unhandled exception of type not derived from std::exception"); } diff --git a/library/cpp/actors/core/event_local.h b/library/cpp/actors/core/event_local.h index 2845aa94dd9..96313cf1e39 100644 --- a/library/cpp/actors/core/event_local.h +++ b/library/cpp/actors/core/event_local.h @@ -3,7 +3,7 @@ #include "event.h" #include "scheduler_cookie.h" #include "event_load.h" -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NActors { template <typename TEv, ui32 TEventType> diff --git a/library/cpp/actors/core/event_pb.h b/library/cpp/actors/core/event_pb.h index d7546b901a0..210d0c2d837 100644 --- a/library/cpp/actors/core/event_pb.h +++ b/library/cpp/actors/core/event_pb.h @@ -268,7 +268,7 @@ namespace NActors { // parse the protobuf TRopeStream stream(iter, size); if (!ev->Record.ParseFromZeroCopyStream(&stream)) { - Y_FAIL("Failed to parse protobuf event type %" PRIu32 " class %s", TEventType, TypeName(ev->Record).data()); + Y_FAIL("Failed to parse protobuf event type %" PRIu32 " class %s", TEventType, TypeName(ev->Record).data()); } } ev->CachedByteSize = input->GetSize(); diff --git a/library/cpp/actors/core/executor_thread.cpp b/library/cpp/actors/core/executor_thread.cpp index 446b651efd2..8c528fd0f4b 100644 --- a/library/cpp/actors/core/executor_thread.cpp +++ b/library/cpp/actors/core/executor_thread.cpp @@ -19,7 +19,7 @@ #include <unistd.h> #endif -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/datetime.h> LWTRACE_USING(ACTORLIB_PROVIDER) @@ -92,7 +92,7 @@ namespace NActors { return "nullptr"; } try { - return TypeName(*t); + return TypeName(*t); } catch (...) { return "unknown-type"; } diff --git a/library/cpp/actors/core/invoke.h b/library/cpp/actors/core/invoke.h index 931a9767ddd..da00728062e 100644 --- a/library/cpp/actors/core/invoke.h +++ b/library/cpp/actors/core/invoke.h @@ -9,8 +9,8 @@ #include <utility> #include <variant> -#include <util/system/type_name.h> - +#include <util/system/type_name.h> + namespace NActors { struct TEvents::TEvInvokeResult @@ -55,7 +55,7 @@ namespace NActors { return std::move(*value); } else { Y_FAIL("unspported return type for TEvInvokeResult: actual# %s != expected# %s", - TypeName(arg.type()).data(), TypeName<T>().data()); + TypeName(arg.type()).data(), TypeName<T>().data()); } }, Result); } diff --git a/library/cpp/actors/core/memory_track.h b/library/cpp/actors/core/memory_track.h index 6035333eeb4..c050aeddcbc 100644 --- a/library/cpp/actors/core/memory_track.h +++ b/library/cpp/actors/core/memory_track.h @@ -2,7 +2,7 @@ #include <vector> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/thread/singleton.h> #define ENABLE_MEMORY_TRACKING diff --git a/library/cpp/actors/interconnect/interconnect_handshake.cpp b/library/cpp/actors/interconnect/interconnect_handshake.cpp index 9ede998d8e7..e76f3998d6b 100644 --- a/library/cpp/actors/interconnect/interconnect_handshake.cpp +++ b/library/cpp/actors/interconnect/interconnect_handshake.cpp @@ -93,8 +93,8 @@ namespace NActors { TMaybe<THolder<TProgramInfo>> ProgramInfo; // filled in in case of successful handshake; even if null TSessionParams Params; bool ResolveTimedOut = false; - THashMap<ui32, TInstant> LastLogNotice; - const TDuration MuteDuration = TDuration::Seconds(15); + THashMap<ui32, TInstant> LastLogNotice; + const TDuration MuteDuration = TDuration::Seconds(15); TInstant Deadline; public: @@ -791,12 +791,12 @@ namespace NActors { if (network) { TInstant now = Now(); - TInstant prevLog = LastLogNotice[PeerNodeId]; - NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG; - if (now - prevLog > MuteDuration) { - logPriority = NActors::NLog::PRI_NOTICE; - LastLogNotice[PeerNodeId] = now; - } + TInstant prevLog = LastLogNotice[PeerNodeId]; + NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG; + if (now - prevLog > MuteDuration) { + logPriority = NActors::NLog::PRI_NOTICE; + LastLogNotice[PeerNodeId] = now; + } LOG_LOG_NET_X(logPriority, PeerNodeId, "network-related error occured on handshake: %s", msg.data()); } else { // calculate log severity based on failure type; permanent failures lead to error log messages @@ -883,12 +883,12 @@ namespace NActors { break; } - auto it = LastLogNotice.find(PeerNodeId); - NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG; - if (it != LastLogNotice.end()) { - LastLogNotice.erase(it); - logPriority = NActors::NLog::PRI_NOTICE; - } + auto it = LastLogNotice.find(PeerNodeId); + NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG; + if (it != LastLogNotice.end()) { + LastLogNotice.erase(it); + logPriority = NActors::NLog::PRI_NOTICE; + } LOG_LOG_IC_X(NActorsServices::INTERCONNECT, "ICH05", logPriority, "connected to peer"); } diff --git a/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp b/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp index 7e2d8ccb948..647d7e0d44f 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp @@ -451,12 +451,12 @@ namespace NActors { break; case TEvHandshakeFail::HANDSHAKE_FAIL_PERMANENT: - TString timeExplanation = " LastSessionDieTime# " + LastSessionDieTime.ToString(); + TString timeExplanation = " LastSessionDieTime# " + LastSessionDieTime.ToString(); if (Session) { InvokeOtherActor(*Session, &TInterconnectSessionTCP::Terminate, TDisconnectReason::HandshakeFailPermanent()); } - TransitToErrorState(ev->Get()->Explanation + timeExplanation, false); + TransitToErrorState(ev->Get()->Explanation + timeExplanation, false); break; } } diff --git a/library/cpp/actors/interconnect/logging.h b/library/cpp/actors/interconnect/logging.h index c429d1cade7..c33654b84a4 100644 --- a/library/cpp/actors/interconnect/logging.h +++ b/library/cpp/actors/interconnect/logging.h @@ -20,13 +20,13 @@ LOG_LOG(::NActors::TActivationContext::AsActorContext(), (priority), (component), "%s " marker " %s", LogPrefix.data(), Sprintf(__VA_ARGS__).data()); \ } while (false) -#define LOG_LOG_NET(priority, NODE_ID, FMT, ...) \ - do { \ - const TActorContext& ctx = ::NActors::TActivationContext::AsActorContext(); \ - LOG_LOG(ctx, (priority), ::NActorsServices::INTERCONNECT_NETWORK, "[%" PRIu32 " <-> %" PRIu32 "] %s", \ - ctx.SelfID.NodeId(), (NODE_ID), Sprintf(FMT, __VA_ARGS__).data()); \ - } while (false) - +#define LOG_LOG_NET(priority, NODE_ID, FMT, ...) \ + do { \ + const TActorContext& ctx = ::NActors::TActivationContext::AsActorContext(); \ + LOG_LOG(ctx, (priority), ::NActorsServices::INTERCONNECT_NETWORK, "[%" PRIu32 " <-> %" PRIu32 "] %s", \ + ctx.SelfID.NodeId(), (NODE_ID), Sprintf(FMT, __VA_ARGS__).data()); \ + } while (false) + #define LOG_EMER_IC(marker, ...) LOG_LOG_IC(::NActorsServices::INTERCONNECT, marker, ::NActors::NLog::PRI_EMER, __VA_ARGS__) #define LOG_ALERT_IC(marker, ...) LOG_LOG_IC(::NActorsServices::INTERCONNECT, marker, ::NActors::NLog::PRI_ALERT, __VA_ARGS__) #define LOG_CRIT_IC(marker, ...) LOG_LOG_IC(::NActorsServices::INTERCONNECT, marker, ::NActors::NLog::PRI_CRIT, __VA_ARGS__) @@ -45,8 +45,8 @@ #define LOG_INFO_IC_SESSION(marker, ...) LOG_LOG_IC(::NActorsServices::INTERCONNECT_SESSION, marker, ::NActors::NLog::PRI_INFO, __VA_ARGS__) #define LOG_DEBUG_IC_SESSION(marker, ...) LOG_LOG_IC(::NActorsServices::INTERCONNECT_SESSION, marker, ::NActors::NLog::PRI_DEBUG, __VA_ARGS__) -#define LOG_NOTICE_NET(NODE_ID, FMT, ...) LOG_LOG_NET(::NActors::NLog::PRI_NOTICE, NODE_ID, FMT, __VA_ARGS__) -#define LOG_DEBUG_NET(NODE_ID, FMT, ...) LOG_LOG_NET(::NActors::NLog::PRI_DEBUG, NODE_ID, FMT, __VA_ARGS__) +#define LOG_NOTICE_NET(NODE_ID, FMT, ...) LOG_LOG_NET(::NActors::NLog::PRI_NOTICE, NODE_ID, FMT, __VA_ARGS__) +#define LOG_DEBUG_NET(NODE_ID, FMT, ...) LOG_LOG_NET(::NActors::NLog::PRI_DEBUG, NODE_ID, FMT, __VA_ARGS__) namespace NActors { class TInterconnectLoggingBase { diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 6fa25b99656..90da777489f 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -890,7 +890,7 @@ namespace NActors { const ui64 localActorId = AllocateLocalId(); if (VERBOSE) { - Cerr << "Register actor " << TypeName(*actor) << " as " << localActorId << ", mailbox: " << hint << "\n"; + Cerr << "Register actor " << TypeName(*actor) << " as " << localActorId << ", mailbox: " << hint << "\n"; } // ok, got mailbox @@ -898,7 +898,7 @@ namespace NActors { // do init const TActorId actorId(FirstNodeId + nodeIndex, poolId, localActorId, hint); - ActorNames[actorId] = TypeName(*actor); + ActorNames[actorId] = TypeName(*actor); RegistrationObserver(*this, parentId ? parentId : CurrentRecipient, actorId); DoActorInit(node->ActorSystem.Get(), actor, actorId, parentId ? parentId : CurrentRecipient); @@ -937,12 +937,12 @@ namespace NActors { const ui64 localActorId = AllocateLocalId(); if (VERBOSE) { - Cerr << "Register actor " << TypeName(*actor) << " as " << localActorId << "\n"; + Cerr << "Register actor " << TypeName(*actor) << " as " << localActorId << "\n"; } mailbox->AttachActor(localActorId, actor); const TActorId actorId(FirstNodeId + nodeIndex, poolId, localActorId, hint); - ActorNames[actorId] = TypeName(*actor); + ActorNames[actorId] = TypeName(*actor); RegistrationObserver(*this, parentId ? parentId : CurrentRecipient, actorId); DoActorInit(node->ActorSystem.Get(), actor, actorId, parentId ? parentId : CurrentRecipient); diff --git a/library/cpp/actors/util/rope.h b/library/cpp/actors/util/rope.h index f5595efbaa6..a506fd000db 100644 --- a/library/cpp/actors/util/rope.h +++ b/library/cpp/actors/util/rope.h @@ -993,130 +993,130 @@ public: } }; -struct TRopeUtils { - static void Memset(TRope::TConstIterator dst, char c, size_t size) { - while (size) { - Y_VERIFY_DEBUG(dst.Valid()); - size_t len = std::min(size, dst.ContiguousSize()); - memset(const_cast<char*>(dst.ContiguousData()), c, len); - dst += len; - size -= len; - } - } - - static void Memcpy(TRope::TConstIterator dst, TRope::TConstIterator src, size_t size) { - while (size) { - Y_VERIFY_DEBUG(dst.Valid() && src.Valid(), - "Invalid iterator in memcpy: dst.Valid() - %" PRIu32 ", src.Valid() - %" PRIu32, - (ui32)dst.Valid(), (ui32)src.Valid()); - size_t len = std::min(size, std::min(dst.ContiguousSize(), src.ContiguousSize())); - memcpy(const_cast<char*>(dst.ContiguousData()), src.ContiguousData(), len); - dst += len; - src += len; - size -= len; - } - } - - static void Memcpy(TRope::TConstIterator dst, const char* src, size_t size) { - while (size) { - Y_VERIFY_DEBUG(dst.Valid()); - size_t len = std::min(size, dst.ContiguousSize()); - memcpy(const_cast<char*>(dst.ContiguousData()), src, len); - size -= len; - dst += len; - src += len; - } - } - - static void Memcpy(char* dst, TRope::TConstIterator src, size_t size) { - while (size) { - Y_VERIFY_DEBUG(src.Valid()); - size_t len = std::min(size, src.ContiguousSize()); - memcpy(dst, src.ContiguousData(), len); - size -= len; - dst += len; - src += len; - } - } - - // copy less or equal to sizeBound bytes, until src is valid - static size_t SafeMemcpy(char* dst, TRope::TIterator src, size_t sizeBound) { - size_t origSize = sizeBound; - while (sizeBound && src.Valid()) { - size_t len = Min(sizeBound, src.ContiguousSize()); - memcpy(dst, src.ContiguousData(), len); - sizeBound -= len; - dst += len; - src += len; - } - return origSize - sizeBound; - } -}; - -template<size_t BLOCK, size_t ALIGN = 16> -class TRopeSlideView { - alignas(ALIGN) char Slide[BLOCK]; // use if distance from current point and next chunk is less than BLOCK - TRope::TIterator Position; // current position at rope - size_t Size; - char* Head; // points to data, it might be current rope chunk or Slide - -private: - void FillBlock() { - size_t chunkSize = Position.ContiguousSize(); - if (chunkSize >= BLOCK) { - Size = chunkSize; - Head = const_cast<char*>(Position.ContiguousData()); - } else { - Size = TRopeUtils::SafeMemcpy(Slide, Position, BLOCK); - Head = Slide; - } - } - -public: - TRopeSlideView(TRope::TIterator position) - : Position(position) - { - FillBlock(); - } - - TRopeSlideView(TRope &rope) - : TRopeSlideView(rope.Begin()) - {} - - // if view on slide then copy slide to rope - void FlushBlock() { - if (Head == Slide) { - TRopeUtils::Memcpy(Position, Head, Size); - } - } - - TRope::TIterator operator+=(size_t amount) { - Position += amount; - FillBlock(); - return Position; - } - - TRope::TIterator GetPosition() const { - return Position; - } - - char* GetHead() const { - return Head; - } - - ui8* GetUi8Head() const { - return reinterpret_cast<ui8*>(Head); - } - - size_t ContiguousSize() const { - return Size; - } - - bool IsOnChunk() const { - return Head != Slide; - } -}; - +struct TRopeUtils { + static void Memset(TRope::TConstIterator dst, char c, size_t size) { + while (size) { + Y_VERIFY_DEBUG(dst.Valid()); + size_t len = std::min(size, dst.ContiguousSize()); + memset(const_cast<char*>(dst.ContiguousData()), c, len); + dst += len; + size -= len; + } + } + + static void Memcpy(TRope::TConstIterator dst, TRope::TConstIterator src, size_t size) { + while (size) { + Y_VERIFY_DEBUG(dst.Valid() && src.Valid(), + "Invalid iterator in memcpy: dst.Valid() - %" PRIu32 ", src.Valid() - %" PRIu32, + (ui32)dst.Valid(), (ui32)src.Valid()); + size_t len = std::min(size, std::min(dst.ContiguousSize(), src.ContiguousSize())); + memcpy(const_cast<char*>(dst.ContiguousData()), src.ContiguousData(), len); + dst += len; + src += len; + size -= len; + } + } + + static void Memcpy(TRope::TConstIterator dst, const char* src, size_t size) { + while (size) { + Y_VERIFY_DEBUG(dst.Valid()); + size_t len = std::min(size, dst.ContiguousSize()); + memcpy(const_cast<char*>(dst.ContiguousData()), src, len); + size -= len; + dst += len; + src += len; + } + } + + static void Memcpy(char* dst, TRope::TConstIterator src, size_t size) { + while (size) { + Y_VERIFY_DEBUG(src.Valid()); + size_t len = std::min(size, src.ContiguousSize()); + memcpy(dst, src.ContiguousData(), len); + size -= len; + dst += len; + src += len; + } + } + + // copy less or equal to sizeBound bytes, until src is valid + static size_t SafeMemcpy(char* dst, TRope::TIterator src, size_t sizeBound) { + size_t origSize = sizeBound; + while (sizeBound && src.Valid()) { + size_t len = Min(sizeBound, src.ContiguousSize()); + memcpy(dst, src.ContiguousData(), len); + sizeBound -= len; + dst += len; + src += len; + } + return origSize - sizeBound; + } +}; + +template<size_t BLOCK, size_t ALIGN = 16> +class TRopeSlideView { + alignas(ALIGN) char Slide[BLOCK]; // use if distance from current point and next chunk is less than BLOCK + TRope::TIterator Position; // current position at rope + size_t Size; + char* Head; // points to data, it might be current rope chunk or Slide + +private: + void FillBlock() { + size_t chunkSize = Position.ContiguousSize(); + if (chunkSize >= BLOCK) { + Size = chunkSize; + Head = const_cast<char*>(Position.ContiguousData()); + } else { + Size = TRopeUtils::SafeMemcpy(Slide, Position, BLOCK); + Head = Slide; + } + } + +public: + TRopeSlideView(TRope::TIterator position) + : Position(position) + { + FillBlock(); + } + + TRopeSlideView(TRope &rope) + : TRopeSlideView(rope.Begin()) + {} + + // if view on slide then copy slide to rope + void FlushBlock() { + if (Head == Slide) { + TRopeUtils::Memcpy(Position, Head, Size); + } + } + + TRope::TIterator operator+=(size_t amount) { + Position += amount; + FillBlock(); + return Position; + } + + TRope::TIterator GetPosition() const { + return Position; + } + + char* GetHead() const { + return Head; + } + + ui8* GetUi8Head() const { + return reinterpret_cast<ui8*>(Head); + } + + size_t ContiguousSize() const { + return Size; + } + + bool IsOnChunk() const { + return Head != Slide; + } +}; + inline TRope TRope::CopySpaceOptimized(TRope&& origin, size_t worstRatioPer1k, TRopeArena& arena) { TRope res; for (TChunk& chunk : origin.Chain) { diff --git a/library/cpp/dbg_output/dump.h b/library/cpp/dbg_output/dump.h index c7efa105ee6..c80ff3fd947 100644 --- a/library/cpp/dbg_output/dump.h +++ b/library/cpp/dbg_output/dump.h @@ -6,7 +6,7 @@ #include "colorscheme.h" #include <util/stream/format.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/generic/hash_set.h> #include <utility> @@ -31,7 +31,7 @@ namespace NPrivate { template <class V> inline void Pointer(const V* v) { if (v) { - *this << DumpRaw("(") << DumpRaw(TypeName(v).data()) << DumpRaw(")") << Hex((size_t)v); + *this << DumpRaw("(") << DumpRaw(TypeName(v).data()) << DumpRaw(")") << Hex((size_t)v); } else { *this << DumpRaw("(") << DumpRaw(TypeName<V>().data()) << DumpRaw("*)nullptr"); } @@ -52,7 +52,7 @@ namespace NPrivate { inline void Pointer(const V* v) { if (v && !Visited.contains((size_t)v)) { Visited.insert((size_t)v); - *this << DumpRaw("(") << DumpRaw(TypeName(v).data()) << DumpRaw(")") << Hex((size_t)v) << DumpRaw(" -> ") << *v; + *this << DumpRaw("(") << DumpRaw(TypeName(v).data()) << DumpRaw(")") << Hex((size_t)v) << DumpRaw(" -> ") << *v; Visited.erase((size_t)v); } else { *this << DumpRaw("(") << DumpRaw(TypeName<V>().data()) << DumpRaw("*)nullptr"); diff --git a/library/cpp/linear_regression/benchmark/main.cpp b/library/cpp/linear_regression/benchmark/main.cpp index 735d41e9888..374d94caabe 100644 --- a/library/cpp/linear_regression/benchmark/main.cpp +++ b/library/cpp/linear_regression/benchmark/main.cpp @@ -5,7 +5,7 @@ #include <util/datetime/base.h> #include <util/datetime/cputimer.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/string/printf.h> diff --git a/library/cpp/messagebus/message.cpp b/library/cpp/messagebus/message.cpp index bfa7ed8e9ba..d0b09a34865 100644 --- a/library/cpp/messagebus/message.cpp +++ b/library/cpp/messagebus/message.cpp @@ -24,7 +24,7 @@ namespace NBus { // TODO: print local flags #ifndef NDEBUG Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point; message type is %s", - MessageType.value_or("unknown").c_str()); + MessageType.value_or("unknown").c_str()); #else Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point"); #endif @@ -83,7 +83,7 @@ namespace NBus { } TString TBusMessage::Describe() const { - return Sprintf("object type: %s, message type: %d", TypeName(*this).data(), int(GetHeader()->Type)); + return Sprintf("object type: %s, message type: %d", TypeName(*this).data(), int(GetHeader()->Type)); } TBusMessage::~TBusMessage() { @@ -184,7 +184,7 @@ namespace NBus { << " flags=" << Flags << " local-flags=" << LocalFlags #ifndef NDEBUG - << " msg-type= " << MessageType.value_or("unknown").c_str() + << " msg-type= " << MessageType.value_or("unknown").c_str() #endif ; return ss.Str(); diff --git a/library/cpp/messagebus/message.h b/library/cpp/messagebus/message.h index 005ca10c652..6e96c964d5c 100644 --- a/library/cpp/messagebus/message.h +++ b/library/cpp/messagebus/message.h @@ -11,12 +11,12 @@ #include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/system/defaults.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/yassert.h> -#include <optional> -#include <typeinfo> - +#include <optional> +#include <typeinfo> + namespace NBus { /////////////////////////////////////////////////////////////////// /// \brief Structure to preserve identity from message to reply @@ -37,10 +37,10 @@ namespace NBus { TInstant RecvTime; #ifndef NDEBUG - std::optional<TString> MessageType; + std::optional<TString> MessageType; #endif - private: + private: // TODO: drop TNetAddr GetNetAddr() const; @@ -91,12 +91,12 @@ namespace NBus { } } - void SetMessageType(const std::type_info& messageTypeInfo) { + void SetMessageType(const std::type_info& messageTypeInfo) { #ifndef NDEBUG Y_VERIFY(!MessageType, "state check"); - MessageType = TypeName(messageTypeInfo); + MessageType = TypeName(messageTypeInfo); #else - Y_UNUSED(messageTypeInfo); + Y_UNUSED(messageTypeInfo); #endif } }; diff --git a/library/cpp/messagebus/rain_check/core/task.cpp b/library/cpp/messagebus/rain_check/core/task.cpp index a098437d534..dd7885ae167 100644 --- a/library/cpp/messagebus/rain_check/core/task.cpp +++ b/library/cpp/messagebus/rain_check/core/task.cpp @@ -2,7 +2,7 @@ #include <library/cpp/messagebus/actor/temp_tls_vector.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/tls.h> using namespace NRainCheck; @@ -208,7 +208,7 @@ ITaskBase* TTaskRunnerBase::CurrentTaskImpl() { } TString TTaskRunnerBase::GetStatusSingleLine() { - return TypeName(*Impl); + return TypeName(*Impl); } bool NRainCheck::AreWeInsideTask() { diff --git a/library/cpp/messagebus/remote_server_session.cpp b/library/cpp/messagebus/remote_server_session.cpp index 6abbf88a606..7daf4098112 100644 --- a/library/cpp/messagebus/remote_server_session.cpp +++ b/library/cpp/messagebus/remote_server_session.cpp @@ -103,8 +103,8 @@ void TRemoteServerSession::InvokeOnMessage(TBusMessagePtrAndHeader& request, TIn ident.RecvTime = request.MessagePtr->RecvTime; #ifndef NDEBUG - auto& message = *request.MessagePtr; - ident.SetMessageType(typeid(message)); + auto& message = *request.MessagePtr; + ident.SetMessageType(typeid(message)); #endif TOnMessageContext context(request.MessagePtr.Release(), ident, this); diff --git a/library/cpp/messagebus/test/helper/object_count_check.h b/library/cpp/messagebus/test/helper/object_count_check.h index 1c4756e58c1..b13ca071853 100644 --- a/library/cpp/messagebus/test/helper/object_count_check.h +++ b/library/cpp/messagebus/test/helper/object_count_check.h @@ -11,7 +11,7 @@ #include <library/cpp/messagebus/scheduler/scheduler.h> #include <util/generic/object_counter.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/stream/output.h> #include <typeinfo> @@ -49,7 +49,7 @@ struct TObjectCountCheck { } void operator()() { - UNIT_ASSERT_VALUES_EQUAL_C(0L, TObjectCounter<T>::ObjectCount(), TypeName<T>()); + UNIT_ASSERT_VALUES_EQUAL_C(0L, TObjectCounter<T>::ObjectCount(), TypeName<T>()); } }; diff --git a/library/cpp/messagebus/ybus.h b/library/cpp/messagebus/ybus.h index de21ad85216..5878b87ca73 100644 --- a/library/cpp/messagebus/ybus.h +++ b/library/cpp/messagebus/ybus.h @@ -30,7 +30,7 @@ #include <util/stream/input.h> #include <util/system/atomic.h> #include <util/system/condvar.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/event.h> #include <util/system/mutex.h> diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp index f727647f7f2..88df97b7273 100644 --- a/library/cpp/on_disk/chunks/chunks_ut.cpp +++ b/library/cpp/on_disk/chunks/chunks_ut.cpp @@ -231,7 +231,7 @@ public: void TestNamedChunkedData() { const TString filename = MakeTempName(nullptr, "named_chunked_data_test"); - TTempFile file(filename); + TTempFile file(filename); { TFixedBufferFileOutput fOut(filename); diff --git a/library/cpp/pop_count/benchmark/main.cpp b/library/cpp/pop_count/benchmark/main.cpp index 41ea3c91cc5..a1b6aa42fd8 100644 --- a/library/cpp/pop_count/benchmark/main.cpp +++ b/library/cpp/pop_count/benchmark/main.cpp @@ -1,6 +1,6 @@ #include <util/stream/output.h> #include <util/datetime/cputimer.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <library/cpp/pop_count/popcount.h> #include <library/cpp/testing/benchmark/bench.h> diff --git a/library/cpp/protobuf/util/path.h b/library/cpp/protobuf/util/path.h index 487f643a2d6..8712f81389e 100644 --- a/library/cpp/protobuf/util/path.h +++ b/library/cpp/protobuf/util/path.h @@ -11,8 +11,8 @@ namespace NProtoBuf { TFieldPath(); TFieldPath(const Descriptor* msgType, const TStringBuf& path); // throws exception if path doesn't exist TFieldPath(const TVector<const FieldDescriptor*>& path); - TFieldPath(const TFieldPath&) = default; - TFieldPath& operator=(const TFieldPath&) = default; + TFieldPath(const TFieldPath&) = default; + TFieldPath& operator=(const TFieldPath&) = default; bool InitUnsafe(const Descriptor* msgType, const TStringBuf path); // noexcept void Init(const Descriptor* msgType, const TStringBuf& path); // throws diff --git a/library/cpp/testing/gtest_extensions/assertions.cpp b/library/cpp/testing/gtest_extensions/assertions.cpp index f390409d1b3..1163da41892 100644 --- a/library/cpp/testing/gtest_extensions/assertions.cpp +++ b/library/cpp/testing/gtest_extensions/assertions.cpp @@ -2,12 +2,12 @@ #include <util/string/builder.h> #include <util/string/split.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NGTest::NInternal { namespace { void FormatActual(const std::exception& err, const TBackTrace* bt, TStringBuilder& out) { - out << "an exception of type " << TypeName(err) << " " + out << "an exception of type " << TypeName(err) << " " << "with message " << TString(err.what()).Quote() << "."; if (bt) { out << "\n Trace: "; diff --git a/library/cpp/testing/unittest/gtest.cpp b/library/cpp/testing/unittest/gtest.cpp index ebad1ea4d67..8be1164afee 100644 --- a/library/cpp/testing/unittest/gtest.cpp +++ b/library/cpp/testing/unittest/gtest.cpp @@ -3,7 +3,7 @@ #include <util/generic/map.h> #include <util/generic/vector.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> using namespace NUnitTest; using namespace NUnitTest::NPrivate; @@ -19,7 +19,7 @@ namespace { } TString TypeId() const override { - return TypeName(*this) + "-" + MyName; + return TypeName(*this) + "-" + MyName; } TString Name() const noexcept override { diff --git a/library/cpp/testing/unittest/registar.cpp b/library/cpp/testing/unittest/registar.cpp index 3679b768ed4..a51f6940763 100644 --- a/library/cpp/testing/unittest/registar.cpp +++ b/library/cpp/testing/unittest/registar.cpp @@ -299,7 +299,7 @@ NUnitTest::TTestBase::TTestBase() noexcept NUnitTest::TTestBase::~TTestBase() = default; TString NUnitTest::TTestBase::TypeId() const { - return TypeName(*this); + return TypeName(*this); } void NUnitTest::TTestBase::SetUp() { diff --git a/library/cpp/testing/unittest/registar.h b/library/cpp/testing/unittest/registar.h index 44517a00924..18652a5178a 100644 --- a/library/cpp/testing/unittest/registar.h +++ b/library/cpp/testing/unittest/registar.h @@ -17,7 +17,7 @@ #include <util/string/printf.h> #include <util/system/defaults.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/spinlock.h> #include <util/system/src_location.h> @@ -282,8 +282,8 @@ private: \ this->AtStart(); #ifndef UT_SKIP_EXCEPTIONS -#define CATCH_REACTION(FN, e, context) this->AddError(("(" + TypeName(e) + ") " + e.what()).data(), context) -#define CATCH_REACTION_BT(FN, e, context) this->AddError(("(" + TypeName(e) + ") " + e.what()).data(), (e.BackTrace() ? e.BackTrace()->PrintToString() : TString()), context) +#define CATCH_REACTION(FN, e, context) this->AddError(("(" + TypeName(e) + ") " + e.what()).data(), context) +#define CATCH_REACTION_BT(FN, e, context) this->AddError(("(" + TypeName(e) + ") " + e.what()).data(), (e.BackTrace() ? e.BackTrace()->PrintToString() : TString()), context) #else #define CATCH_REACTION(FN, e, context) throw #define CATCH_REACTION_BT(FN, e, context) throw diff --git a/library/cpp/threading/atomic/bool.h b/library/cpp/threading/atomic/bool.h index d52544e7626..793673d1e64 100644 --- a/library/cpp/threading/atomic/bool.h +++ b/library/cpp/threading/atomic/bool.h @@ -6,16 +6,16 @@ namespace NAtomic { class TBool { public: TBool() noexcept = default; - + TBool(bool val) noexcept : Val_(val) { } - TBool(const TBool& src) noexcept { - AtomicSet(Val_, AtomicGet(src.Val_)); - } - + TBool(const TBool& src) noexcept { + AtomicSet(Val_, AtomicGet(src.Val_)); + } + operator bool() const noexcept { return AtomicGet(Val_); } diff --git a/library/cpp/tvmauth/client/misc/disk_cache.cpp b/library/cpp/tvmauth/client/misc/disk_cache.cpp index 3c01be4a837..f3fa20dc44f 100644 --- a/library/cpp/tvmauth/client/misc/disk_cache.cpp +++ b/library/cpp/tvmauth/client/misc/disk_cache.cpp @@ -11,7 +11,7 @@ #include <util/stream/str.h> #include <util/system/fs.h> #include <util/system/sysstat.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <exception> diff --git a/util/folder/fts_ut.cpp b/util/folder/fts_ut.cpp index c5d59e35f42..e7cb53e549f 100644 --- a/util/folder/fts_ut.cpp +++ b/util/folder/fts_ut.cpp @@ -6,7 +6,7 @@ #include <library/cpp/threading/future/async.h> #include <util/system/file.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/generic/string.h> class TFtsTest: public TTestBase { diff --git a/util/generic/cast.h b/util/generic/cast.h index 0d4a41f385b..d5eab17cfdc 100644 --- a/util/generic/cast.h +++ b/util/generic/cast.h @@ -4,7 +4,7 @@ #include "yexception.h" #include <util/system/compat.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/unaligned_mem.h> #include <util/system/yassert.h> diff --git a/util/generic/hash.h b/util/generic/hash.h index e46db21fa97..043e8a51428 100644 --- a/util/generic/hash.h +++ b/util/generic/hash.h @@ -4,7 +4,7 @@ #include "mapfindptr.h" #include <util/memory/alloc.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/yassert.h> #include <util/str_stl.h> #include "yexception.h" diff --git a/util/generic/typelist_ut.cpp b/util/generic/typelist_ut.cpp index eeabfa97b17..232ae5ab047 100644 --- a/util/generic/typelist_ut.cpp +++ b/util/generic/typelist_ut.cpp @@ -1,5 +1,5 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include "typelist.h" #include "vector.h" diff --git a/util/generic/yexception.cpp b/util/generic/yexception.cpp index 2ce6c4369da..e7d18cd7788 100644 --- a/util/generic/yexception.cpp +++ b/util/generic/yexception.cpp @@ -2,7 +2,7 @@ #include "yexception.h" #include <util/system/backtrace.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <cxxabi.h> @@ -11,7 +11,7 @@ #include <cstdio> TString FormatExc(const std::exception& exception) { - return TString::Join(TStringBuf("("), TypeName(exception), TStringBuf(") "), exception.what()); + return TString::Join(TStringBuf("("), TypeName(exception), TStringBuf(") "), exception.what()); } TString CurrentExceptionMessage() { diff --git a/util/memory/blob_ut.cpp b/util/memory/blob_ut.cpp index 023f9a04871..4d908b39068 100644 --- a/util/memory/blob_ut.cpp +++ b/util/memory/blob_ut.cpp @@ -2,7 +2,7 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/folder/path.h> #include <util/stream/output.h> #include <util/stream/file.h> diff --git a/util/str_stl.h b/util/str_stl.h index f1e137181d2..dbd6abf67ea 100644 --- a/util/str_stl.h +++ b/util/str_stl.h @@ -7,8 +7,8 @@ #include <util/generic/strbuf.h> #include <util/generic/typetraits.h> -#include <functional> -#include <typeindex> +#include <functional> +#include <typeindex> #include <utility> namespace std { @@ -104,13 +104,13 @@ template <class C, class T, class A> struct hash<std::basic_string<C, T, A>>: ::NHashPrivate::TStringHash<C> { }; -template <> -struct THash<std::type_index> { - inline size_t operator()(const std::type_index& index) const { - return index.hash_code(); - } -}; - +template <> +struct THash<std::type_index> { + inline size_t operator()(const std::type_index& index) const { + return index.hash_code(); + } +}; + namespace NHashPrivate { template <typename T> Y_FORCE_INLINE static size_t HashObject(const T& val) { diff --git a/util/string/split.h b/util/string/split.h index bc46d9e64c6..ca9e1b24e1d 100644 --- a/util/string/split.h +++ b/util/string/split.h @@ -651,58 +651,58 @@ namespace NStringSplitPrivate { using TIteratorOf = typename TIteratorOfImpl<String>::type; template <class String> - class TStringSplitter; - - template <class String> - struct TIterState: public TStringBufOf<String> { - public: + class TStringSplitter; + + template <class String> + struct TIterState: public TStringBufOf<String> { + public: using TStringBufType = TStringBufOf<String>; using TIterator = TIteratorOf<String>; - friend class TStringSplitter<String>; + friend class TStringSplitter<String>; TIterState(const String& string) noexcept - : TStringBufType() - , DelimiterEnd_(std::begin(string)) - , OriginEnd_(std::end(string)) + : TStringBufType() + , DelimiterEnd_(std::begin(string)) + , OriginEnd_(std::end(string)) { } template < typename Other, - typename = std::enable_if_t< - std::is_convertible<Other, TStringBufType>::value>> + typename = std::enable_if_t< + std::is_convertible<Other, TStringBufType>::value>> bool operator==(const Other& toCompare) const { return TStringBufType(*this) == TStringBufType(toCompare); } TIterator TokenStart() const noexcept { - return this->begin(); + return this->begin(); } TIterator TokenDelim() const noexcept { - return this->end(); + return this->end(); } TStringBufType Token() const noexcept { - return *this; + return *this; } TStringBufType Delim() const noexcept { - return MakeStringBuf<TStringBufType>(TokenDelim(), DelimiterEnd_); + return MakeStringBuf<TStringBufType>(TokenDelim(), DelimiterEnd_); } private: - void UpdateParentBuf(TIterator tokenStart, TIterator tokenDelim) noexcept { - *static_cast<TStringBufType*>(this) = MakeStringBuf<TStringBufType>(tokenStart, tokenDelim); + void UpdateParentBuf(TIterator tokenStart, TIterator tokenDelim) noexcept { + *static_cast<TStringBufType*>(this) = MakeStringBuf<TStringBufType>(tokenStart, tokenDelim); } - bool DelimiterIsEmpty() const noexcept { - return TokenDelim() == DelimiterEnd_; + bool DelimiterIsEmpty() const noexcept { + return TokenDelim() == DelimiterEnd_; } - - private: - TIterator DelimiterEnd_; - const TIterator OriginEnd_; + + private: + TIterator DelimiterEnd_; + const TIterator OriginEnd_; }; template <class Base> @@ -829,8 +829,8 @@ namespace NStringSplitPrivate { using TStringType = String; using TChar = typename TStringType::value_type; using TIteratorState = TIterState<TStringType>; - using TStringBufType = typename TIteratorState::TStringBufType; - using TIterator = typename TIteratorState::TIterator; + using TStringBufType = typename TIteratorState::TStringBufType; + using TIterator = typename TIteratorState::TIterator; /** * Base class for all split ranges that actually does the splitting. @@ -841,18 +841,18 @@ namespace NStringSplitPrivate { inline TSplitRangeBase(OtherString&& s, Args&&... args) : String_(std::forward<OtherString>(s)) , State_(String_) - , Delimiter_(std::forward<Args>(args)...) + , Delimiter_(std::forward<Args>(args)...) { } inline TIteratorState* Next() { - if (State_.DelimiterIsEmpty()) { + if (State_.DelimiterIsEmpty()) { return nullptr; } - const auto tokenBegin = State_.DelimiterEnd_; - const auto tokenEnd = Delimiter_.Ptr()->Find(State_.DelimiterEnd_, State_.OriginEnd_); - State_.UpdateParentBuf(tokenBegin, tokenEnd); + const auto tokenBegin = State_.DelimiterEnd_; + const auto tokenEnd = Delimiter_.Ptr()->Find(State_.DelimiterEnd_, State_.OriginEnd_); + State_.UpdateParentBuf(tokenBegin, tokenEnd); return &State_; } @@ -860,7 +860,7 @@ namespace NStringSplitPrivate { private: TStringType String_; TIteratorState State_; - DelimStorage Delimiter_; + DelimStorage Delimiter_; }; template <class Base, class Filter> @@ -888,7 +888,7 @@ namespace NStringSplitPrivate { struct TNonEmptyFilter { template <class TToken> inline bool Accept(const TToken* token) noexcept { - return !token->empty(); + return !token->empty(); } }; @@ -967,8 +967,8 @@ namespace NStringSplitPrivate { --Count; return false; } else if (Count == 1) { - token->DelimiterEnd_ = token->OriginEnd_; - token->UpdateParentBuf(token->TokenStart(), token->DelimiterEnd_); + token->DelimiterEnd_ = token->OriginEnd_; + token->UpdateParentBuf(token->TokenStart(), token->DelimiterEnd_); return false; } return true; diff --git a/util/system/file_ut.cpp b/util/system/file_ut.cpp index 941e6a50f3d..a8ef80b8c04 100644 --- a/util/system/file_ut.cpp +++ b/util/system/file_ut.cpp @@ -1,6 +1,6 @@ #include "file.h" #include "fs.h" -#include "tempfile.h" +#include "tempfile.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/util/system/mktemp.cpp b/util/system/mktemp.cpp index 505b7b4a4b2..c190bd4fe29 100644 --- a/util/system/mktemp.cpp +++ b/util/system/mktemp.cpp @@ -1,4 +1,4 @@ -#include "tempfile.h" +#include "tempfile.h" #include <util/folder/dirut.h> #include <util/generic/yexception.h> @@ -17,57 +17,57 @@ extern "C" int mkstemps(char* path, int slen); -TString MakeTempName(const char* wrkDir, const char* prefix, const char* extension) { +TString MakeTempName(const char* wrkDir, const char* prefix, const char* extension) { #ifndef _win32_ - TString filePath; + TString filePath; if (wrkDir && *wrkDir) { - filePath += wrkDir; + filePath += wrkDir; } else { - filePath += GetSystemTempDir(); + filePath += GetSystemTempDir(); } - if (filePath.back() != '/') { - filePath += '/'; + if (filePath.back() != '/') { + filePath += '/'; } - if (prefix) { - filePath += prefix; + if (prefix) { + filePath += prefix; } - filePath += "XXXXXX"; // mkstemps requirement + filePath += "XXXXXX"; // mkstemps requirement - size_t extensionPartLength = 0; - if (extension && *extension) { - if (extension[0] != '.') { - filePath += '.'; - extensionPartLength += 1; - } - filePath += extension; - extensionPartLength += strlen(extension); + size_t extensionPartLength = 0; + if (extension && *extension) { + if (extension[0] != '.') { + filePath += '.'; + extensionPartLength += 1; + } + filePath += extension; + extensionPartLength += strlen(extension); } - int fd = mkstemps(const_cast<char*>(filePath.data()), extensionPartLength); - if (fd >= 0) { + int fd = mkstemps(const_cast<char*>(filePath.data()), extensionPartLength); + if (fd >= 0) { close(fd); - return filePath; + return filePath; } #else - char tmpDir[MAX_PATH + 1]; // +1 -- for terminating null character - char filePath[MAX_PATH]; + char tmpDir[MAX_PATH + 1]; // +1 -- for terminating null character + char filePath[MAX_PATH]; const char* pDir = 0; - if (wrkDir && *wrkDir) { + if (wrkDir && *wrkDir) { pDir = wrkDir; - } else if (GetTempPath(MAX_PATH + 1, tmpDir)) { - pDir = tmpDir; - } + } else if (GetTempPath(MAX_PATH + 1, tmpDir)) { + pDir = tmpDir; + } - // it always takes up to 3 characters, no more - if (GetTempFileName(pDir, (prefix) ? (prefix) : "yan", 0, filePath)) { - return filePath; - } + // it always takes up to 3 characters, no more + if (GetTempFileName(pDir, (prefix) ? (prefix) : "yan", 0, filePath)) { + return filePath; + } #endif - ythrow TSystemError() << "can not create temp name(" << wrkDir << ", " << prefix << ", " << extension << ")"; + ythrow TSystemError() << "can not create temp name(" << wrkDir << ", " << prefix << ", " << extension << ")"; } diff --git a/util/system/tempfile.cpp b/util/system/tempfile.cpp index a2e9f49eb12..534ccf9b268 100644 --- a/util/system/tempfile.cpp +++ b/util/system/tempfile.cpp @@ -1,9 +1,9 @@ #include "tempfile.h" -TTempFileHandle::TTempFileHandle() - : TTempFile(MakeTempName()) - , TFile(CreateFile()) -{ +TTempFileHandle::TTempFileHandle() + : TTempFile(MakeTempName()) + , TFile(CreateFile()) +{ } TTempFileHandle::TTempFileHandle(const TString& fname) @@ -12,14 +12,14 @@ TTempFileHandle::TTempFileHandle(const TString& fname) { } -TTempFileHandle TTempFileHandle::InCurrentDir(const TString& filePrefix, const TString& extension) { - return TTempFileHandle(MakeTempName(".", filePrefix.c_str(), extension.c_str())); -} - -TTempFileHandle TTempFileHandle::InDir(const TFsPath& dirPath, const TString& filePrefix, const TString& extension) { - return TTempFileHandle(MakeTempName(dirPath.c_str(), filePrefix.c_str(), extension.c_str())); -} - -TFile TTempFileHandle::CreateFile() const { - return TFile(Name(), CreateAlways | RdWr); +TTempFileHandle TTempFileHandle::InCurrentDir(const TString& filePrefix, const TString& extension) { + return TTempFileHandle(MakeTempName(".", filePrefix.c_str(), extension.c_str())); } + +TTempFileHandle TTempFileHandle::InDir(const TFsPath& dirPath, const TString& filePrefix, const TString& extension) { + return TTempFileHandle(MakeTempName(dirPath.c_str(), filePrefix.c_str(), extension.c_str())); +} + +TFile TTempFileHandle::CreateFile() const { + return TFile(Name(), CreateAlways | RdWr); +} diff --git a/util/system/tempfile.h b/util/system/tempfile.h index de249c129da..fb8ec62ec62 100644 --- a/util/system/tempfile.h +++ b/util/system/tempfile.h @@ -3,7 +3,7 @@ #include "fs.h" #include "file.h" -#include <util/folder/path.h> +#include <util/folder/path.h> #include <util/generic/string.h> class TTempFile { @@ -30,21 +30,21 @@ public: TTempFileHandle(); TTempFileHandle(const TString& fname); - static TTempFileHandle InCurrentDir(const TString& filePrefix = "yandex", const TString& extension = "tmp"); - static TTempFileHandle InDir(const TFsPath& dirPath, const TString& filePrefix = "yandex", const TString& extension = "tmp"); - + static TTempFileHandle InCurrentDir(const TString& filePrefix = "yandex", const TString& extension = "tmp"); + static TTempFileHandle InDir(const TFsPath& dirPath, const TString& filePrefix = "yandex", const TString& extension = "tmp"); + private: TFile CreateFile() const; }; - -/* - * Creates a unique temporary filename in specified directory. - * If specified directory is NULL or empty, then system temporary directory is used. - * - * Note, that the function is not race-free, the file is guaranteed to exist at the time the function returns, but not at the time the returned name is first used. - * Throws TSystemError on error. + +/* + * Creates a unique temporary filename in specified directory. + * If specified directory is NULL or empty, then system temporary directory is used. * - * Returned filepath has such format: dir/prefixXXXXXX.extension or dir/prefixXXXXXX - * But win32: dir/preXXXX.tmp (prefix is up to 3 characters, extension is always tmp). - */ -TString MakeTempName(const char* wrkDir = nullptr, const char* prefix = "yandex", const char* extension = "tmp"); + * Note, that the function is not race-free, the file is guaranteed to exist at the time the function returns, but not at the time the returned name is first used. + * Throws TSystemError on error. + * + * Returned filepath has such format: dir/prefixXXXXXX.extension or dir/prefixXXXXXX + * But win32: dir/preXXXX.tmp (prefix is up to 3 characters, extension is always tmp). + */ +TString MakeTempName(const char* wrkDir = nullptr, const char* prefix = "yandex", const char* extension = "tmp"); diff --git a/util/system/tempfile_ut.cpp b/util/system/tempfile_ut.cpp index e4a0923d0bc..3b9ad5fc507 100644 --- a/util/system/tempfile_ut.cpp +++ b/util/system/tempfile_ut.cpp @@ -3,11 +3,11 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/folder/dirut.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/stream/file.h> -#include <algorithm> - +#include <algorithm> + Y_UNIT_TEST_SUITE(TTempFileHandle) { Y_UNIT_TEST(Create) { TString path; @@ -20,132 +20,132 @@ Y_UNIT_TEST_SUITE(TTempFileHandle) { } UNIT_ASSERT(!NFs::Exists(path)); } - - Y_UNIT_TEST(InCurrentDir) { -#ifndef _win32_ - static const TString TEST_PREFIX = "unique_prefix"; -#else - static const TString TEST_PREFIX = "uni"; -#endif - - TString path; - { - TTempFileHandle tmp = TTempFileHandle::InCurrentDir(TEST_PREFIX); - path = tmp.Name(); - UNIT_ASSERT(NFs::Exists(path)); - - TVector<TString> names; - TFsPath(".").ListNames(names); - bool containsFileWithPrefix = std::any_of(names.begin(), names.end(), [&](const TString& name) { - return name.Contains(TEST_PREFIX); - }); - UNIT_ASSERT(containsFileWithPrefix); - } - UNIT_ASSERT(!NFs::Exists(path)); - } - - Y_UNIT_TEST(UseExtensionWithoutDot) { - TString path; - { - TTempFileHandle tmp = TTempFileHandle::InCurrentDir("hello", "world"); - path = tmp.Name(); - UNIT_ASSERT(NFs::Exists(path)); - -#ifndef _win32_ - UNIT_ASSERT(path.Contains("hello")); - UNIT_ASSERT(path.EndsWith(".world")); - UNIT_ASSERT(!path.EndsWith("..world")); -#else - UNIT_ASSERT(path.Contains("hel")); - UNIT_ASSERT(path.EndsWith(".tmp")); -#endif - } - UNIT_ASSERT(!NFs::Exists(path)); - } - - Y_UNIT_TEST(UseExtensionWithDot) { - TString path; - { - TTempFileHandle tmp = TTempFileHandle::InCurrentDir("lorem", ".ipsum"); - path = tmp.Name(); - UNIT_ASSERT(NFs::Exists(path)); - -#ifndef _win32_ - UNIT_ASSERT(path.Contains("lorem")); - UNIT_ASSERT(path.EndsWith(".ipsum")); - UNIT_ASSERT(!path.EndsWith("..ipsum")); -#else - UNIT_ASSERT(path.Contains("lor")); - UNIT_ASSERT(path.EndsWith(".tmp")); -#endif - } - UNIT_ASSERT(!NFs::Exists(path)); - } - - Y_UNIT_TEST(SafeDestructor) { - TString path; - { - path = MakeTempName(); - UNIT_ASSERT(NFs::Exists(path)); - - TTempFileHandle tmp(path); - Y_UNUSED(tmp); - UNIT_ASSERT(NFs::Exists(path)); - - TTempFileHandle anotherTmp(path); - Y_UNUSED(anotherTmp); - UNIT_ASSERT(NFs::Exists(path)); - } - UNIT_ASSERT(!NFs::Exists(path)); - } - - Y_UNIT_TEST(RemovesOpen) { - TString path; - { - TTempFileHandle tmp; - path = tmp.Name(); - tmp.Write("hello world\n", 12); - tmp.FlushData(); - UNIT_ASSERT(NFs::Exists(path)); - UNIT_ASSERT(tmp.IsOpen()); - } - UNIT_ASSERT(!NFs::Exists(path)); - } - - Y_UNIT_TEST(NonExistingDirectory) { - UNIT_ASSERT_EXCEPTION(TTempFileHandle::InDir("nonexsistingdirname"), TSystemError); - } -} - -Y_UNIT_TEST_SUITE(MakeTempName) { - Y_UNIT_TEST(Default) { - TString path; - { - TTempFile tmp(MakeTempName()); - path = tmp.Name(); - - UNIT_ASSERT(!path.Contains('\0')); - UNIT_ASSERT(NFs::Exists(path)); - UNIT_ASSERT(path.EndsWith(".tmp")); - -#ifndef _win32_ - UNIT_ASSERT(path.Contains("yandex")); -#else - UNIT_ASSERT(path.Contains("yan")); -#endif - } - UNIT_ASSERT(!NFs::Exists(path)); - } - - Y_UNIT_TEST(UseNullptr) { - TString path; - { - TTempFile tmp(MakeTempName(nullptr, nullptr, nullptr)); - path = tmp.Name(); - - UNIT_ASSERT(!path.Contains('\0')); - UNIT_ASSERT(NFs::Exists(path)); - } - UNIT_ASSERT(!NFs::Exists(path)); - } + + Y_UNIT_TEST(InCurrentDir) { +#ifndef _win32_ + static const TString TEST_PREFIX = "unique_prefix"; +#else + static const TString TEST_PREFIX = "uni"; +#endif + + TString path; + { + TTempFileHandle tmp = TTempFileHandle::InCurrentDir(TEST_PREFIX); + path = tmp.Name(); + UNIT_ASSERT(NFs::Exists(path)); + + TVector<TString> names; + TFsPath(".").ListNames(names); + bool containsFileWithPrefix = std::any_of(names.begin(), names.end(), [&](const TString& name) { + return name.Contains(TEST_PREFIX); + }); + UNIT_ASSERT(containsFileWithPrefix); + } + UNIT_ASSERT(!NFs::Exists(path)); + } + + Y_UNIT_TEST(UseExtensionWithoutDot) { + TString path; + { + TTempFileHandle tmp = TTempFileHandle::InCurrentDir("hello", "world"); + path = tmp.Name(); + UNIT_ASSERT(NFs::Exists(path)); + +#ifndef _win32_ + UNIT_ASSERT(path.Contains("hello")); + UNIT_ASSERT(path.EndsWith(".world")); + UNIT_ASSERT(!path.EndsWith("..world")); +#else + UNIT_ASSERT(path.Contains("hel")); + UNIT_ASSERT(path.EndsWith(".tmp")); +#endif + } + UNIT_ASSERT(!NFs::Exists(path)); + } + + Y_UNIT_TEST(UseExtensionWithDot) { + TString path; + { + TTempFileHandle tmp = TTempFileHandle::InCurrentDir("lorem", ".ipsum"); + path = tmp.Name(); + UNIT_ASSERT(NFs::Exists(path)); + +#ifndef _win32_ + UNIT_ASSERT(path.Contains("lorem")); + UNIT_ASSERT(path.EndsWith(".ipsum")); + UNIT_ASSERT(!path.EndsWith("..ipsum")); +#else + UNIT_ASSERT(path.Contains("lor")); + UNIT_ASSERT(path.EndsWith(".tmp")); +#endif + } + UNIT_ASSERT(!NFs::Exists(path)); + } + + Y_UNIT_TEST(SafeDestructor) { + TString path; + { + path = MakeTempName(); + UNIT_ASSERT(NFs::Exists(path)); + + TTempFileHandle tmp(path); + Y_UNUSED(tmp); + UNIT_ASSERT(NFs::Exists(path)); + + TTempFileHandle anotherTmp(path); + Y_UNUSED(anotherTmp); + UNIT_ASSERT(NFs::Exists(path)); + } + UNIT_ASSERT(!NFs::Exists(path)); + } + + Y_UNIT_TEST(RemovesOpen) { + TString path; + { + TTempFileHandle tmp; + path = tmp.Name(); + tmp.Write("hello world\n", 12); + tmp.FlushData(); + UNIT_ASSERT(NFs::Exists(path)); + UNIT_ASSERT(tmp.IsOpen()); + } + UNIT_ASSERT(!NFs::Exists(path)); + } + + Y_UNIT_TEST(NonExistingDirectory) { + UNIT_ASSERT_EXCEPTION(TTempFileHandle::InDir("nonexsistingdirname"), TSystemError); + } } + +Y_UNIT_TEST_SUITE(MakeTempName) { + Y_UNIT_TEST(Default) { + TString path; + { + TTempFile tmp(MakeTempName()); + path = tmp.Name(); + + UNIT_ASSERT(!path.Contains('\0')); + UNIT_ASSERT(NFs::Exists(path)); + UNIT_ASSERT(path.EndsWith(".tmp")); + +#ifndef _win32_ + UNIT_ASSERT(path.Contains("yandex")); +#else + UNIT_ASSERT(path.Contains("yan")); +#endif + } + UNIT_ASSERT(!NFs::Exists(path)); + } + + Y_UNIT_TEST(UseNullptr) { + TString path; + { + TTempFile tmp(MakeTempName(nullptr, nullptr, nullptr)); + path = tmp.Name(); + + UNIT_ASSERT(!path.Contains('\0')); + UNIT_ASSERT(NFs::Exists(path)); + } + UNIT_ASSERT(!NFs::Exists(path)); + } +} diff --git a/util/system/type_name.cpp b/util/system/type_name.cpp index 0377da42128..fc578091b38 100644 --- a/util/system/type_name.cpp +++ b/util/system/type_name.cpp @@ -6,7 +6,7 @@ #include <cxxabi.h> #endif -#include "type_name.h" +#include "type_name.h" namespace { @@ -42,19 +42,19 @@ const char* NPrivate::TCppDemangler::Demangle(const char* name) { TString CppDemangle(const TString& name) { return NPrivate::TCppDemangler().Demangle(name.data()); } - -TString TypeName(const std::type_info& typeInfo) { + +TString TypeName(const std::type_info& typeInfo) { TString demangled = CppDemangle(typeInfo.name()); // NOLINT(arcadia-typeid-name-restriction) #if defined(_linux_) || defined(_darwin_) SubstGlobal(demangled, STD_ABI_PREFIX, STD_PREFIX); #endif return demangled; -} - -TString TypeName(const std::type_index& typeIndex) { +} + +TString TypeName(const std::type_index& typeIndex) { TString demangled = CppDemangle(typeIndex.name()); #if defined(_linux_) || defined(_darwin_) SubstGlobal(demangled, STD_ABI_PREFIX, STD_PREFIX); #endif return demangled; -} +} diff --git a/util/system/type_name.h b/util/system/type_name.h index b6619aba3fa..72196542fea 100644 --- a/util/system/type_name.h +++ b/util/system/type_name.h @@ -3,28 +3,28 @@ #include <util/generic/string.h> #include <util/string/subst.h> -#include <typeindex> -#include <typeinfo> - -// Consider using TypeName function family. +#include <typeindex> +#include <typeinfo> + +// Consider using TypeName function family. TString CppDemangle(const TString& name); - -// TypeName function family return human readable type name. - -TString TypeName(const std::type_info& typeInfo); -TString TypeName(const std::type_index& typeInfo); - -// Works for types known at compile-time -// (thus, does not take any inheritance into account) -template <class T> -inline TString TypeName() { - return TypeName(typeid(T)); -} - -// Works for dynamic type, including complex class hierarchies. -// Also, distinguishes between T, T*, T const*, T volatile*, T const volatile*, -// but not T and T const. -template <class T> -inline TString TypeName(const T& t) { - return TypeName(typeid(t)); -} + +// TypeName function family return human readable type name. + +TString TypeName(const std::type_info& typeInfo); +TString TypeName(const std::type_index& typeInfo); + +// Works for types known at compile-time +// (thus, does not take any inheritance into account) +template <class T> +inline TString TypeName() { + return TypeName(typeid(T)); +} + +// Works for dynamic type, including complex class hierarchies. +// Also, distinguishes between T, T*, T const*, T volatile*, T const volatile*, +// but not T and T const. +template <class T> +inline TString TypeName(const T& t) { + return TypeName(typeid(t)); +} diff --git a/util/system/type_name_ut.cpp b/util/system/type_name_ut.cpp index 86597f4232c..95b5a254286 100644 --- a/util/system/type_name_ut.cpp +++ b/util/system/type_name_ut.cpp @@ -1,13 +1,13 @@ -#include "type_name.h" +#include "type_name.h" #include <library/cpp/testing/unittest/registar.h> -#include <util/generic/yexception.h> -#include <util/generic/fwd.h> - -#include <stdexcept> -#include <string> - +#include <util/generic/yexception.h> +#include <util/generic/fwd.h> + +#include <stdexcept> +#include <string> + Y_UNIT_TEST_SUITE(TDemangleTest) { Y_UNIT_TEST(SimpleTest) { // just check it does not crash or leak @@ -16,169 +16,169 @@ Y_UNIT_TEST_SUITE(TDemangleTest) { CppDemangle("Sfsdf$dfsdfTTSFSDF23234::SDFS:FSDFSDF#$%"); } } - -namespace NUtil::NTypeNameTest { - - class TSonde { - // intentionally left empty - }; - - class TRombicHead { - public: - virtual ~TRombicHead() = default; - }; - + +namespace NUtil::NTypeNameTest { + + class TSonde { + // intentionally left empty + }; + + class TRombicHead { + public: + virtual ~TRombicHead() = default; + }; + class TRombicLeftArc: public virtual TRombicHead { - public: - int x; - virtual ~TRombicLeftArc() = default; - }; - + public: + int x; + virtual ~TRombicLeftArc() = default; + }; + class TRombicRightArc: public virtual TRombicHead { - public: - int y; - virtual ~TRombicRightArc() = default; - }; - + public: + int y; + virtual ~TRombicRightArc() = default; + }; + class TRombicTail: public virtual TRombicRightArc, TRombicLeftArc { - public: - virtual ~TRombicTail() = default; - }; - - class TFromThis { - public: - TString GetTypeName() const { - return TypeName(*this); - } - }; -} - -using namespace NUtil::NTypeNameTest; - -Y_UNIT_TEST_SUITE(TypeName) { - Y_UNIT_TEST(FromWellKnownTypes) { - UNIT_ASSERT_VALUES_EQUAL(TypeName<void>(), "void"); -#ifdef _MSC_VER - UNIT_ASSERT_VALUES_EQUAL(TypeName<void*>(), "void * __ptr64"); -#else - UNIT_ASSERT_VALUES_EQUAL(TypeName<void*>(), "void*"); -#endif - UNIT_ASSERT_VALUES_EQUAL(TypeName<int>(), "int"); - UNIT_ASSERT_VALUES_EQUAL(TypeName<double>(), "double"); - -#ifdef _MSC_VER + public: + virtual ~TRombicTail() = default; + }; + + class TFromThis { + public: + TString GetTypeName() const { + return TypeName(*this); + } + }; +} + +using namespace NUtil::NTypeNameTest; + +Y_UNIT_TEST_SUITE(TypeName) { + Y_UNIT_TEST(FromWellKnownTypes) { + UNIT_ASSERT_VALUES_EQUAL(TypeName<void>(), "void"); +#ifdef _MSC_VER + UNIT_ASSERT_VALUES_EQUAL(TypeName<void*>(), "void * __ptr64"); +#else + UNIT_ASSERT_VALUES_EQUAL(TypeName<void*>(), "void*"); +#endif + UNIT_ASSERT_VALUES_EQUAL(TypeName<int>(), "int"); + UNIT_ASSERT_VALUES_EQUAL(TypeName<double>(), "double"); + +#ifdef _MSC_VER UNIT_ASSERT_VALUES_EQUAL(TypeName<std::string>(), "class std::basic_string<char,struct std::char_traits<char>,class std::allocator<char> >"); - UNIT_ASSERT_VALUES_EQUAL(TypeName<std::runtime_error>(), "class std::runtime_error"); -#else + UNIT_ASSERT_VALUES_EQUAL(TypeName<std::runtime_error>(), "class std::runtime_error"); +#else UNIT_ASSERT_VALUES_EQUAL(TypeName<std::string>(), "std::basic_string<char, std::char_traits<char>, std::allocator<char> >"); - UNIT_ASSERT_VALUES_EQUAL(TypeName<std::runtime_error>(), "std::runtime_error"); -#endif - } - - Y_UNIT_TEST(FromArcadiaTypes) { -#ifdef _MSC_VER - UNIT_ASSERT_VALUES_EQUAL(TypeName<yexception>(), "class yexception"); + UNIT_ASSERT_VALUES_EQUAL(TypeName<std::runtime_error>(), "std::runtime_error"); +#endif + } + + Y_UNIT_TEST(FromArcadiaTypes) { +#ifdef _MSC_VER + UNIT_ASSERT_VALUES_EQUAL(TypeName<yexception>(), "class yexception"); UNIT_ASSERT_VALUES_EQUAL(TypeName<TString>(), "class TBasicString<char,struct std::char_traits<char> >"); -#else - UNIT_ASSERT_VALUES_EQUAL(TypeName<yexception>(), "yexception"); +#else + UNIT_ASSERT_VALUES_EQUAL(TypeName<yexception>(), "yexception"); UNIT_ASSERT_VALUES_EQUAL(TypeName<TString>(), "TBasicString<char, std::char_traits<char> >"); -#endif - } - - Y_UNIT_TEST(FromUserTypes) { -#ifdef _MSC_VER - UNIT_ASSERT_VALUES_EQUAL(TypeName<TSonde>(), "class NUtil::NTypeNameTest::TSonde"); - UNIT_ASSERT_VALUES_EQUAL(TypeName<TRombicTail>(), "class NUtil::NTypeNameTest::TRombicTail"); -#else - UNIT_ASSERT_VALUES_EQUAL(TypeName<TSonde>(), "NUtil::NTypeNameTest::TSonde"); - UNIT_ASSERT_VALUES_EQUAL(TypeName<TRombicTail>(), "NUtil::NTypeNameTest::TRombicTail"); -#endif - } - - Y_UNIT_TEST(FromWellKnownValues) { - void* value = (void*)"123"; - const void* constValue = (const void*)"456"; - -#ifdef _MSC_VER - UNIT_ASSERT_VALUES_EQUAL(TypeName(value), "void * __ptr64"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(&value), "void * __ptr64 * __ptr64"); - - UNIT_ASSERT_VALUES_EQUAL(TypeName(constValue), "void const * __ptr64"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(&constValue), "void const * __ptr64 * __ptr64"); -#else - UNIT_ASSERT_VALUES_EQUAL(TypeName(value), "void*"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(&value), "void**"); - - UNIT_ASSERT_VALUES_EQUAL(TypeName(constValue), "void const*"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(&constValue), "void const**"); -#endif - - int zero = 0; - UNIT_ASSERT_VALUES_EQUAL(TypeName(zero), "int"); - - double pi = M_PI; - UNIT_ASSERT_VALUES_EQUAL(TypeName(pi), "double"); - - std::string string; - std::runtime_error err("This is awful"); -#ifdef _MSC_VER +#endif + } + + Y_UNIT_TEST(FromUserTypes) { +#ifdef _MSC_VER + UNIT_ASSERT_VALUES_EQUAL(TypeName<TSonde>(), "class NUtil::NTypeNameTest::TSonde"); + UNIT_ASSERT_VALUES_EQUAL(TypeName<TRombicTail>(), "class NUtil::NTypeNameTest::TRombicTail"); +#else + UNIT_ASSERT_VALUES_EQUAL(TypeName<TSonde>(), "NUtil::NTypeNameTest::TSonde"); + UNIT_ASSERT_VALUES_EQUAL(TypeName<TRombicTail>(), "NUtil::NTypeNameTest::TRombicTail"); +#endif + } + + Y_UNIT_TEST(FromWellKnownValues) { + void* value = (void*)"123"; + const void* constValue = (const void*)"456"; + +#ifdef _MSC_VER + UNIT_ASSERT_VALUES_EQUAL(TypeName(value), "void * __ptr64"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(&value), "void * __ptr64 * __ptr64"); + + UNIT_ASSERT_VALUES_EQUAL(TypeName(constValue), "void const * __ptr64"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(&constValue), "void const * __ptr64 * __ptr64"); +#else + UNIT_ASSERT_VALUES_EQUAL(TypeName(value), "void*"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(&value), "void**"); + + UNIT_ASSERT_VALUES_EQUAL(TypeName(constValue), "void const*"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(&constValue), "void const**"); +#endif + + int zero = 0; + UNIT_ASSERT_VALUES_EQUAL(TypeName(zero), "int"); + + double pi = M_PI; + UNIT_ASSERT_VALUES_EQUAL(TypeName(pi), "double"); + + std::string string; + std::runtime_error err("This is awful"); +#ifdef _MSC_VER UNIT_ASSERT_VALUES_EQUAL(TypeName(string), "class std::basic_string<char,struct std::char_traits<char>,class std::allocator<char> >"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(err), "class std::runtime_error"); -#else + UNIT_ASSERT_VALUES_EQUAL(TypeName(err), "class std::runtime_error"); +#else UNIT_ASSERT_VALUES_EQUAL(TypeName(string), "std::basic_string<char, std::char_traits<char>, std::allocator<char> >"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(err), "std::runtime_error"); -#endif - } - - Y_UNIT_TEST(FromUserValues) { - TSonde sonde; - const TSonde constSonde; - - TRombicTail rombicTail; - TRombicHead& castedTail = rombicTail; - - TFromThis obj; - -#ifdef _MSC_VER - UNIT_ASSERT_VALUES_EQUAL(TypeName(sonde), "class NUtil::NTypeNameTest::TSonde"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(constSonde), "class NUtil::NTypeNameTest::TSonde"); - - UNIT_ASSERT_VALUES_EQUAL(TypeName(rombicTail), "class NUtil::NTypeNameTest::TRombicTail"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(castedTail), "class NUtil::NTypeNameTest::TRombicTail"); - - UNIT_ASSERT_VALUES_EQUAL(obj.GetTypeName(), "class NUtil::NTypeNameTest::TFromThis"); -#else - UNIT_ASSERT_VALUES_EQUAL(TypeName(sonde), "NUtil::NTypeNameTest::TSonde"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(constSonde), "NUtil::NTypeNameTest::TSonde"); - - UNIT_ASSERT_VALUES_EQUAL(TypeName(rombicTail), "NUtil::NTypeNameTest::TRombicTail"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(castedTail), "NUtil::NTypeNameTest::TRombicTail"); - - UNIT_ASSERT_VALUES_EQUAL(obj.GetTypeName(), "NUtil::NTypeNameTest::TFromThis"); -#endif - } - - Y_UNIT_TEST(FromTypeInfo) { - UNIT_ASSERT_VALUES_EQUAL(TypeName(typeid(int)), "int"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(std::type_index(typeid(int))), "int"); - } - - Y_UNIT_TEST(DistinguishPointerQualifiers) { + UNIT_ASSERT_VALUES_EQUAL(TypeName(err), "std::runtime_error"); +#endif + } + + Y_UNIT_TEST(FromUserValues) { + TSonde sonde; + const TSonde constSonde; + + TRombicTail rombicTail; + TRombicHead& castedTail = rombicTail; + + TFromThis obj; + +#ifdef _MSC_VER + UNIT_ASSERT_VALUES_EQUAL(TypeName(sonde), "class NUtil::NTypeNameTest::TSonde"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(constSonde), "class NUtil::NTypeNameTest::TSonde"); + + UNIT_ASSERT_VALUES_EQUAL(TypeName(rombicTail), "class NUtil::NTypeNameTest::TRombicTail"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(castedTail), "class NUtil::NTypeNameTest::TRombicTail"); + + UNIT_ASSERT_VALUES_EQUAL(obj.GetTypeName(), "class NUtil::NTypeNameTest::TFromThis"); +#else + UNIT_ASSERT_VALUES_EQUAL(TypeName(sonde), "NUtil::NTypeNameTest::TSonde"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(constSonde), "NUtil::NTypeNameTest::TSonde"); + + UNIT_ASSERT_VALUES_EQUAL(TypeName(rombicTail), "NUtil::NTypeNameTest::TRombicTail"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(castedTail), "NUtil::NTypeNameTest::TRombicTail"); + + UNIT_ASSERT_VALUES_EQUAL(obj.GetTypeName(), "NUtil::NTypeNameTest::TFromThis"); +#endif + } + + Y_UNIT_TEST(FromTypeInfo) { + UNIT_ASSERT_VALUES_EQUAL(TypeName(typeid(int)), "int"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(std::type_index(typeid(int))), "int"); + } + + Y_UNIT_TEST(DistinguishPointerQualifiers) { char* simplePtr = nullptr; const char* constPtr = nullptr; volatile char* volatilePtr = nullptr; const volatile char* cvPtr = nullptr; - -#ifdef _MSC_VER - UNIT_ASSERT_VALUES_EQUAL(TypeName(simplePtr), "char * __ptr64"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(constPtr), "char const * __ptr64"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(volatilePtr), "char volatile * __ptr64"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(cvPtr), "char const volatile * __ptr64"); -#else - UNIT_ASSERT_VALUES_EQUAL(TypeName(simplePtr), "char*"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(constPtr), "char const*"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(volatilePtr), "char volatile*"); - UNIT_ASSERT_VALUES_EQUAL(TypeName(cvPtr), "char const volatile*"); -#endif - } -} + +#ifdef _MSC_VER + UNIT_ASSERT_VALUES_EQUAL(TypeName(simplePtr), "char * __ptr64"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(constPtr), "char const * __ptr64"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(volatilePtr), "char volatile * __ptr64"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(cvPtr), "char const volatile * __ptr64"); +#else + UNIT_ASSERT_VALUES_EQUAL(TypeName(simplePtr), "char*"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(constPtr), "char const*"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(volatilePtr), "char volatile*"); + UNIT_ASSERT_VALUES_EQUAL(TypeName(cvPtr), "char const volatile*"); +#endif + } +} diff --git a/util/system/ut/ya.make b/util/system/ut/ya.make index 127e7c261e4..04e9e24721c 100644 --- a/util/system/ut/ya.make +++ b/util/system/ut/ya.make @@ -71,7 +71,7 @@ SRCS( system/thread_ut.cpp system/tls_ut.cpp system/types_ut.cpp - system/type_name_ut.cpp + system/type_name_ut.cpp system/user_ut.cpp system/unaligned_mem_ut.cpp system/yassert_ut.cpp diff --git a/util/ya.make b/util/ya.make index 6ebe7e40cfe..b81e175b4ee 100644 --- a/util/ya.make +++ b/util/ya.make @@ -317,7 +317,7 @@ JOIN_SRCS( system/thread.cpp system/tls.cpp system/types.cpp - system/type_name.cpp + system/type_name.cpp system/unaligned_mem.cpp system/user.cpp system/utime.cpp diff --git a/ydb/core/actorlib_impl/proto_ready_actor.h b/ydb/core/actorlib_impl/proto_ready_actor.h index f0b29ec65fd..d70af71a28a 100644 --- a/ydb/core/actorlib_impl/proto_ready_actor.h +++ b/ydb/core/actorlib_impl/proto_ready_actor.h @@ -1,7 +1,7 @@ #pragma once #include <library/cpp/actors/core/actor.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> template <typename TType, TType val> struct TName4Ptr2Member { diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index a2faee326e5..05a76b493e1 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -1205,7 +1205,7 @@ struct TEvBlobStorage { const ui32 Generation; const TInstant Deadline; const ui64 IssuerGuid = RandomNumber<ui64>() | 1; - bool IsMonitored = true; + bool IsMonitored = true; ui32 RestartCounter = 0; TEvBlock(ui64 tabletId, ui32 generation, TInstant deadline) @@ -1227,7 +1227,7 @@ struct TEvBlobStorage { str << "TEvBlock {TabletId# " << TabletId << " Generation# " << Generation << " Deadline# " << Deadline.MilliSeconds() - << " IsMonitored# " << IsMonitored + << " IsMonitored# " << IsMonitored << "}"; return str.Str(); } @@ -1777,7 +1777,7 @@ struct TEvBlobStorage { bool Collect; bool IsMultiCollectAllowed; - bool IsMonitored = true; + bool IsMonitored = true; ui32 RestartCounter = 0; @@ -1837,7 +1837,7 @@ struct TEvBlobStorage { } str << " Hard# " << (Hard ? "true" : "false"); str << " IsMultiCollectAllowed# " << IsMultiCollectAllowed; - str << " IsMonitored# " << IsMonitored; + str << " IsMonitored# " << IsMonitored; if (Keep.Get()) { str << " KeepSize# " << Keep->size() << " {"; for (ui32 i = 0; i < Keep->size(); ++i) { diff --git a/ydb/core/blobstorage/crypto/crypto.cpp b/ydb/core/blobstorage/crypto/crypto.cpp index 0d91e891644..b6e6050443a 100644 --- a/ydb/core/blobstorage/crypto/crypto.cpp +++ b/ydb/core/blobstorage/crypto/crypto.cpp @@ -279,26 +279,26 @@ static void Xor(void* destination, const void* a, const void* b, ui32 size) { ++srcB; } } - -static void Xor(TRope::TIterator destination, TRope::TConstIterator a, const void* b, ui32 size) { - while (size) { - ui8 *dst = (ui8*)destination.ContiguousData(); - const ui8 *srcA = (const ui8*)a.ContiguousData(); - const ui8 *srcB = (const ui8*)b; - const ui32 blockSize = Min(destination.ContiguousSize(), a.ContiguousSize()); - ui32 offset = 0; - for (offset = 0; offset < blockSize && offset < size; offset++) { - *dst = *srcA ^ *srcB; - ++dst; - ++srcA; - ++srcB; - } - destination += offset; - a += offset; - b = srcB; - size -= offset; - } -} + +static void Xor(TRope::TIterator destination, TRope::TConstIterator a, const void* b, ui32 size) { + while (size) { + ui8 *dst = (ui8*)destination.ContiguousData(); + const ui8 *srcA = (const ui8*)a.ContiguousData(); + const ui8 *srcB = (const ui8*)b; + const ui32 blockSize = Min(destination.ContiguousSize(), a.ContiguousSize()); + ui32 offset = 0; + for (offset = 0; offset < blockSize && offset < size; offset++) { + *dst = *srcA ^ *srcB; + ++dst; + ++srcA; + ++srcB; + } + destination += offset; + a += offset; + b = srcB; + size -= offset; + } +} #endif void TStreamCypher::Encrypt(void* destination, const void* source, ui32 size) { @@ -372,44 +372,44 @@ void TStreamCypher::Encrypt(void* destination, const void* source, ui32 size) { #endif } -void TStreamCypher::Encrypt(TRope::TIterator destination, TRope::TIterator source, ui32 size) { -#if ENABLE_ENCRYPTION - if (UnusedBytes) { - if (size <= UnusedBytes) { - Xor(destination, source, Leftover + BLOCK_BYTES - UnusedBytes, size); - UnusedBytes -= size; - return; - } - Xor(destination, source, Leftover + BLOCK_BYTES - UnusedBytes, UnusedBytes); - size -= UnusedBytes; - destination += UnusedBytes; - source += UnusedBytes; - UnusedBytes = 0; - } - - if (Y_UNLIKELY(size == 0)) { // prevent slideView init - return; - } - - constexpr ui32 PACK = 8 * BLOCK_BYTES; - TRopeSlideView<PACK> slideDst(destination); - TRopeSlideView<PACK> slideSrc(source); - - while (size >= PACK) { - Encrypt(slideDst.GetHead(), slideSrc.GetHead(), PACK); - slideDst.FlushBlock(); - - slideDst += PACK; - slideSrc += PACK; - size -= PACK; - } - Encrypt(slideDst.GetHead(), slideSrc.GetHead(), size); - slideDst.FlushBlock(); -#else - TRopeUtils::Memcpy(destination, source, size); -#endif -} - +void TStreamCypher::Encrypt(TRope::TIterator destination, TRope::TIterator source, ui32 size) { +#if ENABLE_ENCRYPTION + if (UnusedBytes) { + if (size <= UnusedBytes) { + Xor(destination, source, Leftover + BLOCK_BYTES - UnusedBytes, size); + UnusedBytes -= size; + return; + } + Xor(destination, source, Leftover + BLOCK_BYTES - UnusedBytes, UnusedBytes); + size -= UnusedBytes; + destination += UnusedBytes; + source += UnusedBytes; + UnusedBytes = 0; + } + + if (Y_UNLIKELY(size == 0)) { // prevent slideView init + return; + } + + constexpr ui32 PACK = 8 * BLOCK_BYTES; + TRopeSlideView<PACK> slideDst(destination); + TRopeSlideView<PACK> slideSrc(source); + + while (size >= PACK) { + Encrypt(slideDst.GetHead(), slideSrc.GetHead(), PACK); + slideDst.FlushBlock(); + + slideDst += PACK; + slideSrc += PACK; + size -= PACK; + } + Encrypt(slideDst.GetHead(), slideSrc.GetHead(), size); + slideDst.FlushBlock(); +#else + TRopeUtils::Memcpy(destination, source, size); +#endif +} + void TStreamCypher::InplaceEncrypt(void *source, ui32 size) { #if ENABLE_ENCRYPTION Encrypt(source, source, size); @@ -419,15 +419,15 @@ void TStreamCypher::InplaceEncrypt(void *source, ui32 size) { #endif } -void TStreamCypher::InplaceEncrypt(TRope::TIterator source, ui32 size) { -#if ENABLE_ENCRYPTION - Encrypt(source, source, size); -#else - Y_UNUSED(source); - Y_UNUSED(size); -#endif -} - +void TStreamCypher::InplaceEncrypt(TRope::TIterator source, ui32 size) { +#if ENABLE_ENCRYPTION + Encrypt(source, source, size); +#else + Y_UNUSED(source); + Y_UNUSED(size); +#endif +} + TStreamCypher::~TStreamCypher() { SecureWipeBuffer((ui8*)Key, GetKeySizeBytes()); SecureWipeBuffer((ui8*)Leftover, sizeof(Leftover)); diff --git a/ydb/core/blobstorage/crypto/crypto.h b/ydb/core/blobstorage/crypto/crypto.h index 45fb647ba43..771097cbaa7 100644 --- a/ydb/core/blobstorage/crypto/crypto.h +++ b/ydb/core/blobstorage/crypto/crypto.h @@ -1,5 +1,5 @@ #pragma once -#include <library/cpp/actors/util/rope.h> +#include <library/cpp/actors/util/rope.h> #include <util/generic/ptr.h> #include <util/system/types.h> @@ -107,9 +107,9 @@ public: void StartMessage(ui64 nonce, ui64 offsetBytes); void EncryptZeroes(void* destination, ui32 size); void Encrypt(void* destination, const void* source, ui32 size); - void Encrypt(TRope::TIterator destination, TRope::TIterator source, ui32 size); + void Encrypt(TRope::TIterator destination, TRope::TIterator source, ui32 size); void InplaceEncrypt(void *source, ui32 size); - void InplaceEncrypt(TRope::TIterator source, ui32 size); + void InplaceEncrypt(TRope::TIterator source, ui32 size); ~TStreamCypher(); }; diff --git a/ydb/core/blobstorage/crypto/crypto_rope_ut.cpp b/ydb/core/blobstorage/crypto/crypto_rope_ut.cpp index 8d4f8c87e54..f675b3b998e 100644 --- a/ydb/core/blobstorage/crypto/crypto_rope_ut.cpp +++ b/ydb/core/blobstorage/crypto/crypto_rope_ut.cpp @@ -1,356 +1,356 @@ #include <ydb/core/blobstorage/crypto/crypto.h> -#include <library/cpp/testing/unittest/registar.h> -#include <util/datetime/cputimer.h> -#include <util/generic/buffer.h> -#include <util/generic/string.h> -#include <util/random/fast.h> -#include <util/stream/format.h> -#include <util/string/printf.h> - -#include <numeric> - +#include <library/cpp/testing/unittest/registar.h> +#include <util/datetime/cputimer.h> +#include <util/generic/buffer.h> +#include <util/generic/string.h> +#include <util/random/fast.h> +#include <util/stream/format.h> +#include <util/string/printf.h> + +#include <numeric> + #include <ydb/core/blobstorage/crypto/ut/ut_helpers.h> - -namespace NKikimr { - -Y_UNIT_TEST_SUITE(TBlobStorageCryptoRope) { - -constexpr size_t ENCIPHER_ALIGN = 16; - -TRope RopeUnitialized(size_t size, size_t chunkSize, size_t chunkAlign = ENCIPHER_ALIGN) { - Y_VERIFY(chunkSize > 0); - TRope rope; - for (size_t i = 0; i < size / chunkSize; i++) { - auto item = MakeIntrusive<TRopeAlignedBufferBackend>(chunkSize, chunkAlign); - rope.Insert(rope.End(), TRope(item)); - } - size_t tail = size % chunkSize; - if (tail) { - auto item = MakeIntrusive<TRopeAlignedBufferBackend>(tail, chunkAlign); - rope.Insert(rope.End(), TRope(item)); - } - return rope; -} - -TRope TestingRope(char *inBuf, size_t size, size_t chunkSize, size_t chunkAlign = ENCIPHER_ALIGN) { - TRope rope = RopeUnitialized(size, chunkSize, chunkAlign); - TRopeUtils::Memcpy(rope.Begin(), inBuf, size); - return rope; -} - -template<size_t SIZE> -TRope TestingRope(size_t chunkSize, size_t chunkAlign = ENCIPHER_ALIGN) { - alignas(16) ui8 in[SIZE]; - for (ui32 i = 0; i < SIZE; ++i) { - in[i] = (ui8)i; - } - return TestingRope(reinterpret_cast<char *>(in), SIZE, chunkSize, chunkAlign); -} - -// FIXME(gleb-kov): ? check if distance between iterator and rope.End() >= size -void CompareRopes(TRope::TIterator lhsBegin, TRope::TIterator rhsBegin, size_t size) { - TString lhs = TString::Uninitialized(size); - lhsBegin.ExtractPlainDataAndAdvance(lhs.Detach(), size); - TString rhs = TString::Uninitialized(size); - rhsBegin.ExtractPlainDataAndAdvance(rhs.Detach(), size); - UNIT_ASSERT_ARRAYS_EQUAL(lhs.data(), rhs.data(), size); -} - -void CompareRopeAndBuf(TRope::TIterator ropeBegin, char* buf, size_t size) { - TString ropeBuf = TString::Uninitialized(size); - TRopeUtils::Memcpy(ropeBuf.Detach(), ropeBegin, size); - UNIT_ASSERT_ARRAYS_EQUAL(ropeBuf.data(), buf, size); -} - -///////////////////////////////////////////////////////////////////////// -// Equality with old code tests -///////////////////////////////////////////////////////////////////////// - - /*Y_UNIT_TEST(TestEqualStreamCypherEncryptZeroes) { - TStreamCypher cypher1; - TStreamCypher cypher2; - - ui64 key = 1; - ui64 nonce = 1; - cypher1.SetKey(key); - cypher2.SetKey(key); - - constexpr size_t SIZE = 5000; - alignas(16) ui8 in[SIZE]; - alignas(16) ui8 out[SIZE]; - for (ui32 i = 0; i < SIZE; ++i) { - in[i] = (ui8)i; - } - - for (size_t chunkSize : {13ul, 113ul, 1001ul, SIZE, 3 * SIZE}) { - TRope outRope = RopeUnitialized(SIZE, chunkSize); - - for (ui32 size = 0; size < SIZE; ++size) { - cypher1.StartMessage(nonce, 0); - cypher2.StartMessage(nonce, 0); - - cypher1.EncryptZeroes(out, size); - cypher2.EncryptZeroes(outRope.Begin(), size); - - CompareRopeAndBuf(outRope.Begin(), reinterpret_cast<char *>(out), size); - } - } - }*/ - - Y_UNIT_TEST(TestEqualInplaceStreamCypher) { - TStreamCypher cypher1; - TStreamCypher cypher2; - - ui64 key = 1; - ui64 nonce = 1; - cypher1.SetKey(key); - cypher2.SetKey(key); - - constexpr size_t SIZE = 5000; - - for (size_t chunkSize : {13ul, 113ul, 1001ul, SIZE, 3 * SIZE}) { - alignas(16) ui8 out[SIZE]; - for (ui32 i = 0; i < SIZE; ++i) { - out[i] = (ui8)i; - } - - TRope outRope = TestingRope(reinterpret_cast<char *>(out), SIZE, chunkSize); - - for (ui32 size = 0; size < SIZE; ++size) { - cypher1.StartMessage(nonce, 0); - cypher2.StartMessage(nonce, 0); - - cypher1.InplaceEncrypt(out, size); - cypher2.InplaceEncrypt(outRope.Begin(), size); - - CompareRopeAndBuf(outRope.Begin(), reinterpret_cast<char *>(out), size); - } - } - } - - Y_UNIT_TEST(TestEqualMixedStreamCypher) { - TStreamCypher cypher1; - TStreamCypher cypher2; - - constexpr size_t SIZE = 5000; - - ui64 key = 1; - ui64 nonce = 1; - cypher1.SetKey(key); - cypher2.SetKey(key); - - alignas(16) ui8 in[SIZE]; - for (ui32 i = 0; i < SIZE; ++i) { - in[i] = (ui8)i; - } - - for (size_t inChunkSize : {13ul, 113ul, SIZE, 3 * SIZE}) { - for (size_t outChunkSize : {13ul, 113ul, SIZE, 3 * SIZE}) { - TRope inRope = TestingRope<SIZE>(inChunkSize); - TRope outRope = RopeUnitialized(SIZE, outChunkSize); - alignas(16) ui8 out[SIZE]; - - for (ui32 size = 1; size < SIZE; ++size) { - ui32 in_offset = size / 7; - cypher1.StartMessage(nonce, 0); - cypher2.StartMessage(nonce, 0); - ui32 size1 = (size - in_offset) % 257; - ui32 size2 = (size - in_offset - size1) % 263; - ui32 size3 = size - size1 - size2 - in_offset; - - cypher1.Encrypt(outRope.Begin(), inRope.Begin() + in_offset, size1); - cypher1.Encrypt(outRope.Begin() + size1, inRope.Begin() + in_offset + size1, size2); - cypher1.Encrypt(outRope.Begin() + size1 + size2, inRope.Begin() + in_offset + size1 + size2, size3); - - cypher2.Encrypt(out, in + in_offset, size1); - cypher2.Encrypt(out + size1, in + in_offset + size1, size2); - cypher2.Encrypt(out + size1 + size2, in + in_offset + size1 + size2, size3); - - CompareRopeAndBuf(outRope.Begin(), reinterpret_cast<char *>(out), size1 + size2 + size3); - } - } - } - } - -///////////////////////////////////////////////////////////////////////// -// Adapted old tests -///////////////////////////////////////////////////////////////////////// - - Y_UNIT_TEST(TestMixedStreamCypher) { - TStreamCypher cypher1; - TStreamCypher cypher2; - - ui64 key = 1; - ui64 nonce = 1; - cypher1.SetKey(key); - - constexpr size_t SIZE = 5000; - size_t chunkSize = 1000; - TRope in = TestingRope<SIZE>(chunkSize); - TRope out = RopeUnitialized(SIZE, chunkSize); - - for (ui32 size = 1; size < SIZE; ++size) { - ui32 in_offset = size / 7; - cypher1.StartMessage(nonce, 0); - ui32 size1 = (size - in_offset) % 257; - ui32 size2 = (size - in_offset - size1) % 263; - ui32 size3 = size - size1 - size2 - in_offset; - - cypher1.Encrypt(out.Begin(), in.Begin() + in_offset, size1); - cypher1.Encrypt(out.Begin() + size1, in.Begin() + in_offset + size1, size2); - cypher1.Encrypt(out.Begin() + size1 + size2, in.Begin() + in_offset + size1 + size2, size3); - - cypher2.SetKey(key); - cypher2.StartMessage(nonce, 0); - cypher2.InplaceEncrypt(out.Begin(), size - in_offset); - - CompareRopes(in.Begin() + in_offset, out.Begin(), size - in_offset); - } - } - - Y_UNIT_TEST(TestOffsetStreamCypher) { - TStreamCypher cypher1; - TStreamCypher cypher2; - - ui64 key = 1; - ui64 nonce = 1; - cypher1.SetKey(key); - - constexpr size_t SIZE = 5000; - size_t chunkSize = 1000; - TRope in = TestingRope<SIZE>(chunkSize); - TRope out = RopeUnitialized(SIZE, chunkSize); - - for (ui32 size = 1; size < SIZE; ++size) { - ui32 in_offset = size / 7; - ui32 size1 = (size - in_offset) % 257; - ui32 size2 = (size - in_offset - size1) % 263; - ui32 size3 = size - size1 - size2 - in_offset; - cypher1.StartMessage(nonce, 0); - cypher1.Encrypt(out.Begin(), in.Begin() + in_offset, size1); - cypher1.StartMessage(nonce, size1); - cypher1.Encrypt(out.Begin() + size1, in.Begin() + in_offset + size1, size2); - cypher1.StartMessage(nonce, size1 + size2); - cypher1.Encrypt(out.Begin() + size1 + size2, in.Begin() + in_offset + size1 + size2, size3); - - cypher2.SetKey(key); - cypher2.StartMessage(nonce, 0); - cypher2.InplaceEncrypt(out.Begin(), size - in_offset); - - CompareRopes(in.Begin() + in_offset, out.Begin(), size - in_offset); - } - } - - Y_UNIT_TEST(TestInplaceStreamCypher) { - TStreamCypher cypher1; - TStreamCypher cypher2; - - ui64 key = 1; - ui64 nonce = 1; - - constexpr size_t SIZE = 5000; - size_t chunkSize = 1000; - TRope in = TestingRope<SIZE>(chunkSize); - TRope out = RopeUnitialized(SIZE, chunkSize); - - for (ui32 size = 1; size < SIZE; ++size) { - cypher1.SetKey(key); - cypher1.StartMessage(nonce, 0); - cypher1.InplaceEncrypt(in.Begin(), size); - - TRopeUtils::Memcpy(out.Begin(), in.Begin(), size); - - cypher2.SetKey(key); - cypher2.StartMessage(nonce, 0); - cypher2.InplaceEncrypt(out.Begin(), size); - - in = TestingRope<SIZE>(chunkSize); - - CompareRopes(in.Begin(), out.Begin(), size); - } - } - - Y_UNIT_TEST(PerfTestStreamCypher) { - TStreamCypher cypher1; - constexpr size_t BUF_SIZE = 256 << 10; - constexpr size_t BUF_ALIGN = 32; - constexpr size_t REPETITIONS = 16; - size_t chunkSize = 2 << 20; - - Cout << Endl; - - auto testCase = {std::make_pair(0,0), {4, 0}, {8, 0}, {0, 4}, {0, 8}, {4, 8}, {8, 8}}; - for (auto s : testCase) { - size_t inShift = s.first; - size_t outShift = s.second; - const size_t size = BUF_SIZE; - - Cout << "size# " << HumanReadableSize(size, SF_BYTES); - Cout << " inShift# " << LeftPad(inShift, 2); - Cout << " outShift# " << LeftPad(outShift, 2); - - TVector<TDuration> times; - times.reserve(REPETITIONS); - - for (ui32 i = 0; i < REPETITIONS; ++i) { - TAlignedBuf inBuf(BUF_SIZE, BUF_ALIGN); - for (ui32 i = 0; i < size; ++i) { - inBuf.Data()[i] = (ui8)i; - } - - TRope inRope = TestingRope(reinterpret_cast<char*>(inBuf.Data()), BUF_SIZE, chunkSize, BUF_ALIGN); - TRope outRope = RopeUnitialized(BUF_SIZE, chunkSize, BUF_ALIGN); - - ui64 key = 123; - ui64 nonce = 1; - cypher1.SetKey(key); - cypher1.StartMessage(nonce, 0); - - TSimpleTimer timer; - cypher1.Encrypt(outRope.Begin() + outShift, inRope.Begin() + inShift, size - BUF_ALIGN); - times.push_back(timer.Get()); - } - TDuration min_time = *std::min_element(times.begin(), times.end()); - Cout << " max_speed# " << HumanReadableSize(size / min_time.SecondsFloat(), SF_QUANTITY) << "/s"; - TDuration avg_time = std::accumulate(times.begin(), times.end(), TDuration()) / times.size(); - Cout << " avg_speed# " << HumanReadableSize(size / avg_time.SecondsFloat(), SF_QUANTITY) << "/s"; - Cout << Endl; - } - } - - Y_UNIT_TEST(UnalignedTestStreamCypher) { - constexpr size_t BUF_ALIGN = 8; - size_t chunkSize = 1000; - - TStreamCypher cypher; - - for (size_t size = 151; size < 6923; size = 2*size + 1) { - auto testCase = {std::make_pair(0,0), {8, 0}}; - for (auto s : testCase) { - size_t inShift = s.first; - - Cout << " inShift# " << LeftPad(inShift, 2) << " "; - - TAlignedBuf inBuf(size, BUF_ALIGN); - ui8 *in = inBuf.Data() + inShift; - TReallyFastRng32 rng(692); - for (ui32 i = 0; i < size; ++i) { - in[i] = rng.GenRand() % 256; - } - - TRope inRope = TestingRope(reinterpret_cast<char*>(in), size, chunkSize, BUF_ALIGN); - TRope outRope = RopeUnitialized(size, chunkSize, BUF_ALIGN); - - ui64 key = 123; - ui64 nonce = 1; - cypher.SetKey(key); - cypher.StartMessage(nonce, 0); - - cypher.Encrypt(outRope.Begin(), inRope.Begin() + inShift, size - inShift); - } - } - Cout << Endl; - } - -} -} // namespace NKikimr + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(TBlobStorageCryptoRope) { + +constexpr size_t ENCIPHER_ALIGN = 16; + +TRope RopeUnitialized(size_t size, size_t chunkSize, size_t chunkAlign = ENCIPHER_ALIGN) { + Y_VERIFY(chunkSize > 0); + TRope rope; + for (size_t i = 0; i < size / chunkSize; i++) { + auto item = MakeIntrusive<TRopeAlignedBufferBackend>(chunkSize, chunkAlign); + rope.Insert(rope.End(), TRope(item)); + } + size_t tail = size % chunkSize; + if (tail) { + auto item = MakeIntrusive<TRopeAlignedBufferBackend>(tail, chunkAlign); + rope.Insert(rope.End(), TRope(item)); + } + return rope; +} + +TRope TestingRope(char *inBuf, size_t size, size_t chunkSize, size_t chunkAlign = ENCIPHER_ALIGN) { + TRope rope = RopeUnitialized(size, chunkSize, chunkAlign); + TRopeUtils::Memcpy(rope.Begin(), inBuf, size); + return rope; +} + +template<size_t SIZE> +TRope TestingRope(size_t chunkSize, size_t chunkAlign = ENCIPHER_ALIGN) { + alignas(16) ui8 in[SIZE]; + for (ui32 i = 0; i < SIZE; ++i) { + in[i] = (ui8)i; + } + return TestingRope(reinterpret_cast<char *>(in), SIZE, chunkSize, chunkAlign); +} + +// FIXME(gleb-kov): ? check if distance between iterator and rope.End() >= size +void CompareRopes(TRope::TIterator lhsBegin, TRope::TIterator rhsBegin, size_t size) { + TString lhs = TString::Uninitialized(size); + lhsBegin.ExtractPlainDataAndAdvance(lhs.Detach(), size); + TString rhs = TString::Uninitialized(size); + rhsBegin.ExtractPlainDataAndAdvance(rhs.Detach(), size); + UNIT_ASSERT_ARRAYS_EQUAL(lhs.data(), rhs.data(), size); +} + +void CompareRopeAndBuf(TRope::TIterator ropeBegin, char* buf, size_t size) { + TString ropeBuf = TString::Uninitialized(size); + TRopeUtils::Memcpy(ropeBuf.Detach(), ropeBegin, size); + UNIT_ASSERT_ARRAYS_EQUAL(ropeBuf.data(), buf, size); +} + +///////////////////////////////////////////////////////////////////////// +// Equality with old code tests +///////////////////////////////////////////////////////////////////////// + + /*Y_UNIT_TEST(TestEqualStreamCypherEncryptZeroes) { + TStreamCypher cypher1; + TStreamCypher cypher2; + + ui64 key = 1; + ui64 nonce = 1; + cypher1.SetKey(key); + cypher2.SetKey(key); + + constexpr size_t SIZE = 5000; + alignas(16) ui8 in[SIZE]; + alignas(16) ui8 out[SIZE]; + for (ui32 i = 0; i < SIZE; ++i) { + in[i] = (ui8)i; + } + + for (size_t chunkSize : {13ul, 113ul, 1001ul, SIZE, 3 * SIZE}) { + TRope outRope = RopeUnitialized(SIZE, chunkSize); + + for (ui32 size = 0; size < SIZE; ++size) { + cypher1.StartMessage(nonce, 0); + cypher2.StartMessage(nonce, 0); + + cypher1.EncryptZeroes(out, size); + cypher2.EncryptZeroes(outRope.Begin(), size); + + CompareRopeAndBuf(outRope.Begin(), reinterpret_cast<char *>(out), size); + } + } + }*/ + + Y_UNIT_TEST(TestEqualInplaceStreamCypher) { + TStreamCypher cypher1; + TStreamCypher cypher2; + + ui64 key = 1; + ui64 nonce = 1; + cypher1.SetKey(key); + cypher2.SetKey(key); + + constexpr size_t SIZE = 5000; + + for (size_t chunkSize : {13ul, 113ul, 1001ul, SIZE, 3 * SIZE}) { + alignas(16) ui8 out[SIZE]; + for (ui32 i = 0; i < SIZE; ++i) { + out[i] = (ui8)i; + } + + TRope outRope = TestingRope(reinterpret_cast<char *>(out), SIZE, chunkSize); + + for (ui32 size = 0; size < SIZE; ++size) { + cypher1.StartMessage(nonce, 0); + cypher2.StartMessage(nonce, 0); + + cypher1.InplaceEncrypt(out, size); + cypher2.InplaceEncrypt(outRope.Begin(), size); + + CompareRopeAndBuf(outRope.Begin(), reinterpret_cast<char *>(out), size); + } + } + } + + Y_UNIT_TEST(TestEqualMixedStreamCypher) { + TStreamCypher cypher1; + TStreamCypher cypher2; + + constexpr size_t SIZE = 5000; + + ui64 key = 1; + ui64 nonce = 1; + cypher1.SetKey(key); + cypher2.SetKey(key); + + alignas(16) ui8 in[SIZE]; + for (ui32 i = 0; i < SIZE; ++i) { + in[i] = (ui8)i; + } + + for (size_t inChunkSize : {13ul, 113ul, SIZE, 3 * SIZE}) { + for (size_t outChunkSize : {13ul, 113ul, SIZE, 3 * SIZE}) { + TRope inRope = TestingRope<SIZE>(inChunkSize); + TRope outRope = RopeUnitialized(SIZE, outChunkSize); + alignas(16) ui8 out[SIZE]; + + for (ui32 size = 1; size < SIZE; ++size) { + ui32 in_offset = size / 7; + cypher1.StartMessage(nonce, 0); + cypher2.StartMessage(nonce, 0); + ui32 size1 = (size - in_offset) % 257; + ui32 size2 = (size - in_offset - size1) % 263; + ui32 size3 = size - size1 - size2 - in_offset; + + cypher1.Encrypt(outRope.Begin(), inRope.Begin() + in_offset, size1); + cypher1.Encrypt(outRope.Begin() + size1, inRope.Begin() + in_offset + size1, size2); + cypher1.Encrypt(outRope.Begin() + size1 + size2, inRope.Begin() + in_offset + size1 + size2, size3); + + cypher2.Encrypt(out, in + in_offset, size1); + cypher2.Encrypt(out + size1, in + in_offset + size1, size2); + cypher2.Encrypt(out + size1 + size2, in + in_offset + size1 + size2, size3); + + CompareRopeAndBuf(outRope.Begin(), reinterpret_cast<char *>(out), size1 + size2 + size3); + } + } + } + } + +///////////////////////////////////////////////////////////////////////// +// Adapted old tests +///////////////////////////////////////////////////////////////////////// + + Y_UNIT_TEST(TestMixedStreamCypher) { + TStreamCypher cypher1; + TStreamCypher cypher2; + + ui64 key = 1; + ui64 nonce = 1; + cypher1.SetKey(key); + + constexpr size_t SIZE = 5000; + size_t chunkSize = 1000; + TRope in = TestingRope<SIZE>(chunkSize); + TRope out = RopeUnitialized(SIZE, chunkSize); + + for (ui32 size = 1; size < SIZE; ++size) { + ui32 in_offset = size / 7; + cypher1.StartMessage(nonce, 0); + ui32 size1 = (size - in_offset) % 257; + ui32 size2 = (size - in_offset - size1) % 263; + ui32 size3 = size - size1 - size2 - in_offset; + + cypher1.Encrypt(out.Begin(), in.Begin() + in_offset, size1); + cypher1.Encrypt(out.Begin() + size1, in.Begin() + in_offset + size1, size2); + cypher1.Encrypt(out.Begin() + size1 + size2, in.Begin() + in_offset + size1 + size2, size3); + + cypher2.SetKey(key); + cypher2.StartMessage(nonce, 0); + cypher2.InplaceEncrypt(out.Begin(), size - in_offset); + + CompareRopes(in.Begin() + in_offset, out.Begin(), size - in_offset); + } + } + + Y_UNIT_TEST(TestOffsetStreamCypher) { + TStreamCypher cypher1; + TStreamCypher cypher2; + + ui64 key = 1; + ui64 nonce = 1; + cypher1.SetKey(key); + + constexpr size_t SIZE = 5000; + size_t chunkSize = 1000; + TRope in = TestingRope<SIZE>(chunkSize); + TRope out = RopeUnitialized(SIZE, chunkSize); + + for (ui32 size = 1; size < SIZE; ++size) { + ui32 in_offset = size / 7; + ui32 size1 = (size - in_offset) % 257; + ui32 size2 = (size - in_offset - size1) % 263; + ui32 size3 = size - size1 - size2 - in_offset; + cypher1.StartMessage(nonce, 0); + cypher1.Encrypt(out.Begin(), in.Begin() + in_offset, size1); + cypher1.StartMessage(nonce, size1); + cypher1.Encrypt(out.Begin() + size1, in.Begin() + in_offset + size1, size2); + cypher1.StartMessage(nonce, size1 + size2); + cypher1.Encrypt(out.Begin() + size1 + size2, in.Begin() + in_offset + size1 + size2, size3); + + cypher2.SetKey(key); + cypher2.StartMessage(nonce, 0); + cypher2.InplaceEncrypt(out.Begin(), size - in_offset); + + CompareRopes(in.Begin() + in_offset, out.Begin(), size - in_offset); + } + } + + Y_UNIT_TEST(TestInplaceStreamCypher) { + TStreamCypher cypher1; + TStreamCypher cypher2; + + ui64 key = 1; + ui64 nonce = 1; + + constexpr size_t SIZE = 5000; + size_t chunkSize = 1000; + TRope in = TestingRope<SIZE>(chunkSize); + TRope out = RopeUnitialized(SIZE, chunkSize); + + for (ui32 size = 1; size < SIZE; ++size) { + cypher1.SetKey(key); + cypher1.StartMessage(nonce, 0); + cypher1.InplaceEncrypt(in.Begin(), size); + + TRopeUtils::Memcpy(out.Begin(), in.Begin(), size); + + cypher2.SetKey(key); + cypher2.StartMessage(nonce, 0); + cypher2.InplaceEncrypt(out.Begin(), size); + + in = TestingRope<SIZE>(chunkSize); + + CompareRopes(in.Begin(), out.Begin(), size); + } + } + + Y_UNIT_TEST(PerfTestStreamCypher) { + TStreamCypher cypher1; + constexpr size_t BUF_SIZE = 256 << 10; + constexpr size_t BUF_ALIGN = 32; + constexpr size_t REPETITIONS = 16; + size_t chunkSize = 2 << 20; + + Cout << Endl; + + auto testCase = {std::make_pair(0,0), {4, 0}, {8, 0}, {0, 4}, {0, 8}, {4, 8}, {8, 8}}; + for (auto s : testCase) { + size_t inShift = s.first; + size_t outShift = s.second; + const size_t size = BUF_SIZE; + + Cout << "size# " << HumanReadableSize(size, SF_BYTES); + Cout << " inShift# " << LeftPad(inShift, 2); + Cout << " outShift# " << LeftPad(outShift, 2); + + TVector<TDuration> times; + times.reserve(REPETITIONS); + + for (ui32 i = 0; i < REPETITIONS; ++i) { + TAlignedBuf inBuf(BUF_SIZE, BUF_ALIGN); + for (ui32 i = 0; i < size; ++i) { + inBuf.Data()[i] = (ui8)i; + } + + TRope inRope = TestingRope(reinterpret_cast<char*>(inBuf.Data()), BUF_SIZE, chunkSize, BUF_ALIGN); + TRope outRope = RopeUnitialized(BUF_SIZE, chunkSize, BUF_ALIGN); + + ui64 key = 123; + ui64 nonce = 1; + cypher1.SetKey(key); + cypher1.StartMessage(nonce, 0); + + TSimpleTimer timer; + cypher1.Encrypt(outRope.Begin() + outShift, inRope.Begin() + inShift, size - BUF_ALIGN); + times.push_back(timer.Get()); + } + TDuration min_time = *std::min_element(times.begin(), times.end()); + Cout << " max_speed# " << HumanReadableSize(size / min_time.SecondsFloat(), SF_QUANTITY) << "/s"; + TDuration avg_time = std::accumulate(times.begin(), times.end(), TDuration()) / times.size(); + Cout << " avg_speed# " << HumanReadableSize(size / avg_time.SecondsFloat(), SF_QUANTITY) << "/s"; + Cout << Endl; + } + } + + Y_UNIT_TEST(UnalignedTestStreamCypher) { + constexpr size_t BUF_ALIGN = 8; + size_t chunkSize = 1000; + + TStreamCypher cypher; + + for (size_t size = 151; size < 6923; size = 2*size + 1) { + auto testCase = {std::make_pair(0,0), {8, 0}}; + for (auto s : testCase) { + size_t inShift = s.first; + + Cout << " inShift# " << LeftPad(inShift, 2) << " "; + + TAlignedBuf inBuf(size, BUF_ALIGN); + ui8 *in = inBuf.Data() + inShift; + TReallyFastRng32 rng(692); + for (ui32 i = 0; i < size; ++i) { + in[i] = rng.GenRand() % 256; + } + + TRope inRope = TestingRope(reinterpret_cast<char*>(in), size, chunkSize, BUF_ALIGN); + TRope outRope = RopeUnitialized(size, chunkSize, BUF_ALIGN); + + ui64 key = 123; + ui64 nonce = 1; + cypher.SetKey(key); + cypher.StartMessage(nonce, 0); + + cypher.Encrypt(outRope.Begin(), inRope.Begin() + inShift, size - inShift); + } + } + Cout << Endl; + } + +} +} // namespace NKikimr diff --git a/ydb/core/blobstorage/crypto/ut/ut_helpers.h b/ydb/core/blobstorage/crypto/ut/ut_helpers.h index e9695541df2..b5b25097540 100644 --- a/ydb/core/blobstorage/crypto/ut/ut_helpers.h +++ b/ydb/core/blobstorage/crypto/ut/ut_helpers.h @@ -1,6 +1,6 @@ #pragma once -#include <library/cpp/actors/util/rope.h> +#include <library/cpp/actors/util/rope.h> #include <library/cpp/testing/unittest/registar.h> #include <util/system/align.h> @@ -57,23 +57,23 @@ public: } }; -class TRopeAlignedBufferBackend : public IRopeChunkBackend { - TAlignedBuf Buffer; - -public: - TRopeAlignedBufferBackend(size_t size, size_t align) - : Buffer(size, align) - {} - - TData GetData() const override { - return {reinterpret_cast<const char *>(Buffer.Data()), Buffer.Size()}; - } - - size_t GetCapacity() const override { - return Buffer.Size(); - } -}; - +class TRopeAlignedBufferBackend : public IRopeChunkBackend { + TAlignedBuf Buffer; + +public: + TRopeAlignedBufferBackend(size_t size, size_t align) + : Buffer(size, align) + {} + + TData GetData() const override { + return {reinterpret_cast<const char *>(Buffer.Data()), Buffer.Size()}; + } + + size_t GetCapacity() const override { + return Buffer.Size(); + } +}; + void inline Print(const ui8* out, size_t size) { for (ui32 i = 0; i < size; ++i) { if (i % 16 == 0) { diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp index 73afac8121f..e8943e0f844 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_mon.cpp @@ -1,36 +1,36 @@ #include "dsproxy_mon.h" -static const TVector<float> Percentiles1 = {1.0f}; -static const TVector<float> Percentiles4 = {0.5f, 0.9f, 0.95f, 1.0f}; - +static const TVector<float> Percentiles1 = {1.0f}; +static const TVector<float> Percentiles4 = {0.5f, 0.9f, 0.95f, 1.0f}; + namespace NKikimr { TBlobStorageGroupProxyMon::TBlobStorageGroupProxyMon(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, const TIntrusivePtr<NMonitoring::TDynamicCounters>& percentileCounters, const TIntrusivePtr<NMonitoring::TDynamicCounters>& overviewCounters, const TIntrusivePtr<TBlobStorageGroupInfo>& info, - const TIntrusivePtr<TDsProxyNodeMon> &nodeMon, - bool constructLimited) + const TIntrusivePtr<TDsProxyNodeMon> &nodeMon, + bool constructLimited) : NodeMon(nodeMon) , Counters(counters) - , PercentileCounters(percentileCounters) - , ResponseGroup(percentileCounters->GetSubgroup("subsystem", "response")) + , PercentileCounters(percentileCounters) + , ResponseGroup(percentileCounters->GetSubgroup("subsystem", "response")) , LatencyOverviewGroup(overviewCounters->GetSubgroup("subsystem", "latency")) , EventGroup(Counters->GetSubgroup("subsystem", "event")) , HandoffGroup(Counters->GetSubgroup("subsystem", "handoff")) , ActiveRequestsGroup(Counters->GetSubgroup("subsystem", "requests")) { if (info) { - const TBlobStorageGroupInfo::TDynamicInfo& dyn = info->GetDynamicInfo(); - GroupIdGen = (ui64(dyn.GroupId) << 32) | dyn.GroupGeneration; + const TBlobStorageGroupInfo::TDynamicInfo& dyn = info->GetDynamicInfo(); + GroupIdGen = (ui64(dyn.GroupId) << 32) | dyn.GroupGeneration; } - BlockResponseTime.Initialize(ResponseGroup, "event", "block", "Response in millisec", Percentiles1); - - if (!constructLimited) { - BecomeFull(); - } + BlockResponseTime.Initialize(ResponseGroup, "event", "block", "Response in millisec", Percentiles1); + if (!constructLimited) { + BecomeFull(); + } + // event counters EventPut = EventGroup->GetCounter("EvPut", true); EventPutBytes = EventGroup->GetCounter("EvPutBytes", true); @@ -103,38 +103,38 @@ TBlobStorageGroupProxyMon::TBlobStorageGroupProxyMon(const TIntrusivePtr<NMonito RespStatPatch.emplace(respStatGroup->GetSubgroup("request", "patch")); } -void TBlobStorageGroupProxyMon::BecomeFull() { - if (IsLimitedMon) { - ThroughputGroup = PercentileCounters->GetSubgroup("subsystem", "throughput"); +void TBlobStorageGroupProxyMon::BecomeFull() { + if (IsLimitedMon) { + ThroughputGroup = PercentileCounters->GetSubgroup("subsystem", "throughput"); PutTabletLogThroughput.reset(new TThroughputMeter(4, ThroughputGroup, "event", "putTabletLog", "bytes per second", Percentiles4)); PutAsyncBlobThroughput.reset(new TThroughputMeter(4, ThroughputGroup, "event", "putAsyncBlob", "bytes per second", Percentiles4)); PutUserDataThroughput.reset(new TThroughputMeter(4, ThroughputGroup, "event", "putUserData", "bytes per second", Percentiles4)); PutThroughput.reset(new TThroughputMeter(4, ThroughputGroup, "event", "any", "bytes per second", Percentiles4)); - - PutResponseTime.Initialize(ResponseGroup, "event", "put", "Response in millisec", Percentiles4); - - TIntrusivePtr<NMonitoring::TDynamicCounters> putTabletLogGroup = - ResponseGroup->GetSubgroup("event", "putTabletLog"); - - PutTabletLogResponseTime.Initialize(ResponseGroup, "event", "putTabletLogAll", "ms", Percentiles1); - - PutTabletLogResponseTime256.Initialize(putTabletLogGroup, "size", "256", "Response in millisec", Percentiles1); - PutTabletLogResponseTime512.Initialize(putTabletLogGroup, "size", "512", "Response in millisec", Percentiles1); - - PutAsyncBlobResponseTime.Initialize(ResponseGroup, "event", "putAsyncBlob", "Response in millisec", Percentiles1); - PutUserDataResponseTime.Initialize(ResponseGroup, "event", "putUserData", "Response in millisec", Percentiles1); - - GetResponseTime.Initialize(ResponseGroup, "event", "get", "Response in millisec", Percentiles1); - - DiscoverResponseTime.Initialize(ResponseGroup, "event", "discover", "Response in millisec", Percentiles1); - IndexRestoreGetResponseTime.Initialize(ResponseGroup, "event", "indexRestoreGet", "Response in millisec", - Percentiles1); - RangeResponseTime.Initialize(ResponseGroup, "event", "range", "Response in millisec", Percentiles1); + + PutResponseTime.Initialize(ResponseGroup, "event", "put", "Response in millisec", Percentiles4); + + TIntrusivePtr<NMonitoring::TDynamicCounters> putTabletLogGroup = + ResponseGroup->GetSubgroup("event", "putTabletLog"); + + PutTabletLogResponseTime.Initialize(ResponseGroup, "event", "putTabletLogAll", "ms", Percentiles1); + + PutTabletLogResponseTime256.Initialize(putTabletLogGroup, "size", "256", "Response in millisec", Percentiles1); + PutTabletLogResponseTime512.Initialize(putTabletLogGroup, "size", "512", "Response in millisec", Percentiles1); + + PutAsyncBlobResponseTime.Initialize(ResponseGroup, "event", "putAsyncBlob", "Response in millisec", Percentiles1); + PutUserDataResponseTime.Initialize(ResponseGroup, "event", "putUserData", "Response in millisec", Percentiles1); + + GetResponseTime.Initialize(ResponseGroup, "event", "get", "Response in millisec", Percentiles1); + + DiscoverResponseTime.Initialize(ResponseGroup, "event", "discover", "Response in millisec", Percentiles1); + IndexRestoreGetResponseTime.Initialize(ResponseGroup, "event", "indexRestoreGet", "Response in millisec", + Percentiles1); + RangeResponseTime.Initialize(ResponseGroup, "event", "range", "Response in millisec", Percentiles1); PatchResponseTime.Initialize(ResponseGroup, "event", "patch", "Response in millisec", Percentiles1); - } - IsLimitedMon = false; -} - + } + IsLimitedMon = false; +} + void TBlobStorageGroupProxyMon::SerializeToWhiteboard(NKikimrWhiteboard::TBSGroupStateInfo& pb, ui32 groupId) const { NKikimrWhiteboard::EFlag flag = NKikimrWhiteboard::EFlag::Green; @@ -163,9 +163,9 @@ void TBlobStorageGroupProxyMon::SerializeToWhiteboard(NKikimrWhiteboard::TBSGrou } bool TBlobStorageGroupProxyMon::GetGroupIdGen(ui32 *groupId, ui32 *groupGen) const { - ui64 value = GroupIdGen; - if (value != Max<ui64>()) { - const ui64 mask = Max<ui32>(); + ui64 value = GroupIdGen; + if (value != Max<ui64>()) { + const ui64 mask = Max<ui32>(); *groupId = (value >> 32) & mask; *groupGen = value & mask; return true; @@ -197,10 +197,10 @@ void TBlobStorageGroupProxyMon::Update() { } void TBlobStorageGroupProxyMon::ThroughputUpdate() { - if (!IsLimitedMon) { - for (auto *sensor : {&PutTabletLogThroughput, &PutAsyncBlobThroughput, &PutUserDataThroughput, &PutThroughput}) { + if (!IsLimitedMon) { + for (auto *sensor : {&PutTabletLogThroughput, &PutAsyncBlobThroughput, &PutUserDataThroughput, &PutThroughput}) { sensor->get()->UpdateHistogram(); - } + } } } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_mon.h b/ydb/core/blobstorage/dsproxy/dsproxy_mon.h index 941bea6664a..cbfc5afd82c 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_mon.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_mon.h @@ -139,9 +139,9 @@ public: protected: TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - TIntrusivePtr<NMonitoring::TDynamicCounters> PercentileCounters; - TIntrusivePtr<NMonitoring::TDynamicCounters> ResponseGroup; - ui64 GroupIdGen = Max<ui64>(); // group id:group gen + TIntrusivePtr<NMonitoring::TDynamicCounters> PercentileCounters; + TIntrusivePtr<NMonitoring::TDynamicCounters> ResponseGroup; + ui64 GroupIdGen = Max<ui64>(); // group id:group gen std::atomic<bool> IsLimitedMon = ATOMIC_VAR_INIT(true); TIntrusivePtr<NMonitoring::TDynamicCounters> ThroughputGroup; @@ -267,13 +267,13 @@ public: const TIntrusivePtr<NMonitoring::TDynamicCounters>& percentileCounters, const TIntrusivePtr<NMonitoring::TDynamicCounters>& overviewCounters, const TIntrusivePtr<TBlobStorageGroupInfo>& info, - const TIntrusivePtr<TDsProxyNodeMon> &nodeMon, - bool isLimitedMon); + const TIntrusivePtr<TDsProxyNodeMon> &nodeMon, + bool isLimitedMon); bool GetGroupIdGen(ui32 *groupId, ui32 *groupGen) const; - void BecomeFull(); - + void BecomeFull(); + void SerializeToWhiteboard(NKikimrWhiteboard::TBSGroupStateInfo& pb, ui32 groupId) const; void CountPutEvent(ui32 size) { @@ -287,16 +287,16 @@ public: void CountThroughput(NKikimrBlobStorage::EPutHandleClass cls, ui32 size) { switch (cls) { case NKikimrBlobStorage::EPutHandleClass::TabletLog: - PutTabletLogThroughput->Count(size); + PutTabletLogThroughput->Count(size); break; case NKikimrBlobStorage::EPutHandleClass::AsyncBlob: - PutAsyncBlobThroughput->Count(size); + PutAsyncBlobThroughput->Count(size); break; case NKikimrBlobStorage::EPutHandleClass::UserData: - PutUserDataThroughput->Count(size); + PutUserDataThroughput->Count(size); break; } - PutThroughput->Count(size); + PutThroughput->Count(size); } void CountPutPesponseTime(TPDiskCategory::EDeviceType type, NKikimrBlobStorage::EPutHandleClass cls, ui32 size, diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp index d9e0e2b3202..3e589ba4476 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp +++ b/ydb/core/blobstorage/dsproxy/dsproxy_put.cpp @@ -456,7 +456,7 @@ public: , PutImpl(info, state, ev, mon, enableRequestMod3x3ForMinLatecy) , WaitingVDiskResponseCount(info->GetTotalVDisksNum()) , Deadline(ev->Deadline) - , StartTime(now) + , StartTime(now) , HandleClass(ev->HandleClass) , ReportedBytes(0) , TimeStatsEnabled(timeStatsEnabled) @@ -502,7 +502,7 @@ public: , WaitingVDiskResponseCount(info->GetTotalVDisksNum()) , IsManyPuts(true) , Deadline(TInstant::Zero()) - , StartTime(now) + , StartTime(now) , HandleClass(handleClass) , ReportedBytes(0) , TimeStatsEnabled(timeStatsEnabled) @@ -631,7 +631,7 @@ public: << " Group# " << Info->GroupID << " BlobIDs# " << BlobIdSequenceToString() << " Not answered in " - << (TActivationContext::Now() - StartTime).Seconds() << " seconds"); + << (TActivationContext::Now() - StartTime).Seconds() << " seconds"); if (TInstant::Now() > Deadline) { ErrorReason = "Deadline exceeded"; ReplyAndDie(NKikimrProto::DEADLINE); diff --git a/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp b/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp index d2d39a35504..696c7d3bf34 100644 --- a/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut/dsproxy_put_ut.cpp @@ -41,7 +41,7 @@ void TestPutMaxPartCountOnHandoff(TErasureType::EErasureSpecies erasureSpecies) TIntrusivePtr<NMonitoring::TDynamicCounters> counters(new NMonitoring::TDynamicCounters()); TIntrusivePtr<TDsProxyNodeMon> nodeMon(new TDsProxyNodeMon(counters, true)); TIntrusivePtr<TBlobStorageGroupProxyMon> mon(new TBlobStorageGroupProxyMon(counters, counters, counters, - group.GetInfo(), nodeMon, false)); + group.GetInfo(), nodeMon, false)); TLogContext logCtx(NKikimrServices::BS_PROXY_PUT, false); logCtx.LogAcc.IsLogEnabled = false; @@ -177,7 +177,7 @@ struct TTestPutAllOk { , Data(AlphaData(DataSize)) , Counters(new NMonitoring::TDynamicCounters()) , NodeMon(new TDsProxyNodeMon(Counters, true)) - , Mon(new TBlobStorageGroupProxyMon(Counters, Counters, Counters, Group.GetInfo(), NodeMon, false)) + , Mon(new TBlobStorageGroupProxyMon(Counters, Counters, Counters, Group.GetInfo(), NodeMon, false)) , LogCtx(NKikimrServices::BS_PROXY_PUT, false) , PartSets(BlobCount) { diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index e3c74cce7b4..27ca28e2f21 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -428,22 +428,22 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { } }; - void BlockGroup(TTestBasicRuntime& runtime, TActorId sender, ui64 tabletId, ui32 groupId, ui32 generation, bool isMonitored, - NKikimrProto::EReplyStatus expectAnsver = NKikimrProto::EReplyStatus::OK) { + void BlockGroup(TTestBasicRuntime& runtime, TActorId sender, ui64 tabletId, ui32 groupId, ui32 generation, bool isMonitored, + NKikimrProto::EReplyStatus expectAnsver = NKikimrProto::EReplyStatus::OK) { auto request = std::make_unique<TEvBlobStorage::TEvBlock>(tabletId, generation, TInstant::Max()); - request->IsMonitored = isMonitored; + request->IsMonitored = isMonitored; SendToBsProxy(runtime, sender, groupId, request.release()); auto reply = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvBlockResult>(sender); UNIT_ASSERT_VALUES_EQUAL(reply->Get()->Status, expectAnsver); } - void CollectGroup(TTestBasicRuntime& runtime, TActorId sender, ui64 tabletId, ui32 groupId, bool isMonitored, - NKikimrProto::EReplyStatus expectAnsver = NKikimrProto::EReplyStatus::OK) { + void CollectGroup(TTestBasicRuntime& runtime, TActorId sender, ui64 tabletId, ui32 groupId, bool isMonitored, + NKikimrProto::EReplyStatus expectAnsver = NKikimrProto::EReplyStatus::OK) { auto request = std::make_unique<TEvBlobStorage::TEvCollectGarbage>(tabletId, Max<ui32>(), Max<ui32>(), ui32(0), true, Max<ui32>(), Max<ui32>(), nullptr, nullptr, TInstant::Max(), true, true); - request->IsMonitored = isMonitored; + request->IsMonitored = isMonitored; SendToBsProxy(runtime, sender, groupId, request.release()); auto reply = runtime.GrabEdgeEventRethrow<TEvBlobStorage::TEvCollectGarbageResult>(sender); UNIT_ASSERT_VALUES_EQUAL(reply->Get()->Status, expectAnsver); @@ -460,9 +460,9 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { ui64 tabletId = 1234; ui32 generation = 1; - BlockGroup(runtime, sender0, tabletId, groupId, generation, true); - BlockGroup(runtime, sender0, tabletId, groupId, generation, true, NKikimrProto::EReplyStatus::RACE); - BlockGroup(runtime, sender0, tabletId, groupId, generation-1, true, NKikimrProto::EReplyStatus::RACE); + BlockGroup(runtime, sender0, tabletId, groupId, generation, true); + BlockGroup(runtime, sender0, tabletId, groupId, generation, true, NKikimrProto::EReplyStatus::RACE); + BlockGroup(runtime, sender0, tabletId, groupId, generation-1, true, NKikimrProto::EReplyStatus::RACE); auto describePool = DescribeStoragePool(runtime, DOMAIN_ID, "test_storage"); { @@ -470,17 +470,17 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { RemoveStoragePool(runtime, DOMAIN_ID, describePool); ++generation; - BlockGroup(runtime, sender0, tabletId, groupId, generation++, true); + BlockGroup(runtime, sender0, tabletId, groupId, generation++, true); } ++generation; - BlockGroup(runtime, sender0, tabletId, groupId, generation++, true); + BlockGroup(runtime, sender0, tabletId, groupId, generation++, true); auto stateStorage = runtime.GetAppData().DomainsInfo->GetDefaultStateStorageGroup(DOMAIN_ID); RebootTablet(runtime, MakeBSControllerID(stateStorage), sender0, sender0.NodeId() - runtime.GetNodeId(0)); ++generation; - BlockGroup(runtime, sender0, tabletId, groupId, generation++, true, NKikimrProto::EReplyStatus::NO_GROUP); + BlockGroup(runtime, sender0, tabletId, groupId, generation++, true, NKikimrProto::EReplyStatus::NO_GROUP); } CUSTOM_UNIT_TEST(TestSendToInvalidGroupId) { @@ -494,10 +494,10 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { ui64 tabletId = 1234; ui32 generation = 1; - BlockGroup(runtime, sender, tabletId, groupId, generation, true, NKikimrProto::ERROR); + BlockGroup(runtime, sender, tabletId, groupId, generation, true, NKikimrProto::ERROR); Put(runtime, sender, groupId, TLogoBlobID(tabletId, generation, 0, 0, 5, 0), "hello", NKikimrProto::EReplyStatus::ERROR); - CollectGroup(runtime, sender, tabletId, groupId, true, NKikimrProto::EReplyStatus::ERROR); + CollectGroup(runtime, sender, tabletId, groupId, true, NKikimrProto::EReplyStatus::ERROR); } CUSTOM_UNIT_TEST(TestBlockEncriptedGroup) { @@ -512,135 +512,135 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { ui64 tabletId = 1234; ui32 generation = 1; - BlockGroup(runtime, sender0, tabletId, groupId, generation, true); + BlockGroup(runtime, sender0, tabletId, groupId, generation, true); Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation, 0, 0, 5, 0), "hello", NKikimrProto::EReplyStatus::BLOCKED); Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 5, 0), "hello"); - BlockGroup(runtime, sender1, tabletId, groupId, generation+2, true); + BlockGroup(runtime, sender1, tabletId, groupId, generation+2, true); Put(runtime, sender1, groupId, TLogoBlobID(tabletId, generation+2, 0, 0, 10, 0), "hellohello", NKikimrProto::EReplyStatus::ERROR); Put(runtime, sender1, groupId, TLogoBlobID(tabletId, generation+3, 0, 0, 10, 0), "hellohello", NKikimrProto::EReplyStatus::ERROR); Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 11, 0), "hello_again", NKikimrProto::EReplyStatus::BLOCKED); - CollectGroup(runtime, sender1, tabletId, groupId, true); - } - - void AssertMonitoringExists(TTestBasicRuntime& runtime, ui32 nodeIdx, TString groupName) { - auto rootStats = runtime.GetDynamicCounters(nodeIdx); - auto stats = GetServiceCounters(rootStats, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", groupName); - auto responseStats = stats->GetSubgroup("subsystem", "response"); - auto putTabletStats = responseStats->GetSubgroup("event", "putTabletLog"); - - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "putTabletLogAll"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "putAsyncBlob"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "putUserData"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "get"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "block"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "discover"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "indexRestoreGet"), nullptr); - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "range"), nullptr); - - UNIT_ASSERT_UNEQUAL(putTabletStats->FindSubgroup("size", "256"), nullptr); - UNIT_ASSERT_UNEQUAL(putTabletStats->FindSubgroup("size", "512"), nullptr); - } - - void AssertMonitoringDoesNotExist(TTestBasicRuntime& runtime, ui32 nodeIdx, TString groupName) { - auto rootStats = runtime.GetDynamicCounters(nodeIdx); - auto stats = GetServiceCounters(rootStats, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", groupName); - auto responseStats = stats->GetSubgroup("subsystem", "response"); - auto putTabletStats = responseStats->GetSubgroup("event", "putTabletLog"); - - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "putTabletLogAll"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "putAsyncBlob"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "putUserData"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "get"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "discover"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "indexRestoreGet"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "range"), nullptr); - - UNIT_ASSERT_VALUES_EQUAL(putTabletStats->FindSubgroup("size", "256"), nullptr); - UNIT_ASSERT_VALUES_EQUAL(putTabletStats->FindSubgroup("size", "512"), nullptr); - - // always send BlockResponseTime - UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "block"), nullptr); - } - - CUSTOM_UNIT_TEST(TestLimitedKeylessGroupThenNoMonitoring) { - TTestBasicRuntime runtime(2, false); - Setup(runtime, "", nullptr); - - auto sender0 = runtime.AllocateEdgeActor(0); - auto sender1 = runtime.AllocateEdgeActor(1); - - CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); - - ui32 generation = 1; - ui64 tabletId = 1234; - ui32 groupId = GetGroupFromPool(runtime, DOMAIN_ID, "test_storage"); - TString name = Sprintf("%09" PRIu32, groupId); - - BlockGroup(runtime, sender0, tabletId, groupId, generation, true); - - Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation, 0, 0, 5, 0), "hello", NKikimrProto::EReplyStatus::BLOCKED); - Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 5, 0), "hello"); - - BlockGroup(runtime, sender1, tabletId, groupId, generation+2, true); - Put(runtime, sender1, groupId, TLogoBlobID(tabletId, generation+2, 0, 0, 10, 0), "hellohello", NKikimrProto::EReplyStatus::ERROR); - Put(runtime, sender1, groupId, TLogoBlobID(tabletId, generation+3, 0, 0, 10, 0), "hellohello", NKikimrProto::EReplyStatus::ERROR); - - Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 11, 0), "hello_again", NKikimrProto::EReplyStatus::BLOCKED); - - CollectGroup(runtime, sender1, tabletId, groupId, true); - - AssertMonitoringDoesNotExist(runtime, 1, name); // expect IsLimitedKeyLess on node 1 - } - - CUSTOM_UNIT_TEST(TestUnmonitoredEventsThenNoMonitorings) { - TTestBasicRuntime runtime(1, false); - Setup(runtime, "", nullptr); - - auto sender0 = runtime.AllocateEdgeActor(0); - - CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); - - ui32 generation = 1; - ui64 tabletId = 1234; - ui32 groupId = GetGroupFromPool(runtime, DOMAIN_ID, "test_storage"); - TString name = Sprintf("%09" PRIu32, groupId); - - BlockGroup(runtime, sender0, tabletId, groupId, generation, false); - CollectGroup(runtime, sender0, tabletId, groupId, false); - - AssertMonitoringDoesNotExist(runtime, 0, name); - - BlockGroup(runtime, sender0, tabletId, groupId, generation + 2, true); - AssertMonitoringExists(runtime, 0, name); - - BlockGroup(runtime, sender0, tabletId, groupId, generation + 3, false); - AssertMonitoringExists(runtime, 0, name); // it cannot disappear - } - - CUSTOM_UNIT_TEST(TestSendUsefulMonitoring) { - TTestBasicRuntime runtime(2, false); - Setup(runtime, "", nullptr); - - auto sender0 = runtime.AllocateEdgeActor(0); - auto sender1 = runtime.AllocateEdgeActor(1); - - CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); - - ui32 generation = 1; - ui64 tabletId = 1234; - ui32 groupId = GetGroupFromPool(runtime, DOMAIN_ID, "test_storage"); - TString name = Sprintf("%09" PRIu32, groupId); - - Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 5, 0), "hello"); - CollectGroup(runtime, sender1, tabletId, groupId, true); - - AssertMonitoringExists(runtime, 0, name); + CollectGroup(runtime, sender1, tabletId, groupId, true); } + void AssertMonitoringExists(TTestBasicRuntime& runtime, ui32 nodeIdx, TString groupName) { + auto rootStats = runtime.GetDynamicCounters(nodeIdx); + auto stats = GetServiceCounters(rootStats, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", groupName); + auto responseStats = stats->GetSubgroup("subsystem", "response"); + auto putTabletStats = responseStats->GetSubgroup("event", "putTabletLog"); + + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "putTabletLogAll"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "putAsyncBlob"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "putUserData"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "get"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "block"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "discover"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "indexRestoreGet"), nullptr); + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "range"), nullptr); + + UNIT_ASSERT_UNEQUAL(putTabletStats->FindSubgroup("size", "256"), nullptr); + UNIT_ASSERT_UNEQUAL(putTabletStats->FindSubgroup("size", "512"), nullptr); + } + + void AssertMonitoringDoesNotExist(TTestBasicRuntime& runtime, ui32 nodeIdx, TString groupName) { + auto rootStats = runtime.GetDynamicCounters(nodeIdx); + auto stats = GetServiceCounters(rootStats, "dsproxy_percentile")->GetSubgroup("blobstorageproxy", groupName); + auto responseStats = stats->GetSubgroup("subsystem", "response"); + auto putTabletStats = responseStats->GetSubgroup("event", "putTabletLog"); + + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "putTabletLogAll"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "putAsyncBlob"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "putUserData"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "get"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "discover"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "indexRestoreGet"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(responseStats->FindSubgroup("event", "range"), nullptr); + + UNIT_ASSERT_VALUES_EQUAL(putTabletStats->FindSubgroup("size", "256"), nullptr); + UNIT_ASSERT_VALUES_EQUAL(putTabletStats->FindSubgroup("size", "512"), nullptr); + + // always send BlockResponseTime + UNIT_ASSERT_UNEQUAL(responseStats->FindSubgroup("event", "block"), nullptr); + } + + CUSTOM_UNIT_TEST(TestLimitedKeylessGroupThenNoMonitoring) { + TTestBasicRuntime runtime(2, false); + Setup(runtime, "", nullptr); + + auto sender0 = runtime.AllocateEdgeActor(0); + auto sender1 = runtime.AllocateEdgeActor(1); + + CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); + + ui32 generation = 1; + ui64 tabletId = 1234; + ui32 groupId = GetGroupFromPool(runtime, DOMAIN_ID, "test_storage"); + TString name = Sprintf("%09" PRIu32, groupId); + + BlockGroup(runtime, sender0, tabletId, groupId, generation, true); + + Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation, 0, 0, 5, 0), "hello", NKikimrProto::EReplyStatus::BLOCKED); + Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 5, 0), "hello"); + + BlockGroup(runtime, sender1, tabletId, groupId, generation+2, true); + Put(runtime, sender1, groupId, TLogoBlobID(tabletId, generation+2, 0, 0, 10, 0), "hellohello", NKikimrProto::EReplyStatus::ERROR); + Put(runtime, sender1, groupId, TLogoBlobID(tabletId, generation+3, 0, 0, 10, 0), "hellohello", NKikimrProto::EReplyStatus::ERROR); + + Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 11, 0), "hello_again", NKikimrProto::EReplyStatus::BLOCKED); + + CollectGroup(runtime, sender1, tabletId, groupId, true); + + AssertMonitoringDoesNotExist(runtime, 1, name); // expect IsLimitedKeyLess on node 1 + } + + CUSTOM_UNIT_TEST(TestUnmonitoredEventsThenNoMonitorings) { + TTestBasicRuntime runtime(1, false); + Setup(runtime, "", nullptr); + + auto sender0 = runtime.AllocateEdgeActor(0); + + CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); + + ui32 generation = 1; + ui64 tabletId = 1234; + ui32 groupId = GetGroupFromPool(runtime, DOMAIN_ID, "test_storage"); + TString name = Sprintf("%09" PRIu32, groupId); + + BlockGroup(runtime, sender0, tabletId, groupId, generation, false); + CollectGroup(runtime, sender0, tabletId, groupId, false); + + AssertMonitoringDoesNotExist(runtime, 0, name); + + BlockGroup(runtime, sender0, tabletId, groupId, generation + 2, true); + AssertMonitoringExists(runtime, 0, name); + + BlockGroup(runtime, sender0, tabletId, groupId, generation + 3, false); + AssertMonitoringExists(runtime, 0, name); // it cannot disappear + } + + CUSTOM_UNIT_TEST(TestSendUsefulMonitoring) { + TTestBasicRuntime runtime(2, false); + Setup(runtime, "", nullptr); + + auto sender0 = runtime.AllocateEdgeActor(0); + auto sender1 = runtime.AllocateEdgeActor(1); + + CreateStoragePool(runtime, DOMAIN_ID, "test_storage", "pool-kind-1"); + + ui32 generation = 1; + ui64 tabletId = 1234; + ui32 groupId = GetGroupFromPool(runtime, DOMAIN_ID, "test_storage"); + TString name = Sprintf("%09" PRIu32, groupId); + + Put(runtime, sender0, groupId, TLogoBlobID(tabletId, generation+1, 0, 0, 5, 0), "hello"); + CollectGroup(runtime, sender1, tabletId, groupId, true); + + AssertMonitoringExists(runtime, 0, name); + } + CUSTOM_UNIT_TEST(TestGivenPDiskFormatedWithGuid1AndCreatedWithGuid2WhenYardInitThenError) { TTempDir tempDir; TTestBasicRuntime runtime(2, false); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index d00fc0f2203..4854c9e36e0 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -2011,7 +2011,7 @@ void TPDisk::TryTrimChunk(bool prevDone, ui64 trimmedSize) { } if (ChunkBeingTrimmed) { // Initiate trim of next part of chunk - const ui64 trimStep = (Keeper.GetTrimmedFreeChunkCount() > 100 ? 2 << 20 : 32 << 20); + const ui64 trimStep = (Keeper.GetTrimmedFreeChunkCount() > 100 ? 2 << 20 : 32 << 20); ui64 trimSize = Min<ui64>(Format.ChunkSize - TrimOffset, trimStep); TChunkTrim* trim = ReqCreator.CreateChunkTrim(ChunkBeingTrimmed, TrimOffset, trimSize); InputRequest(trim); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_req_creator.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_req_creator.h index af5402a4f81..c69e32ddf89 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_req_creator.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_req_creator.h @@ -9,7 +9,7 @@ #include "blobstorage_pdisk_mon.h" #include "blobstorage_pdisk_requestimpl.h" -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NKikimr::NPDisk { diff --git a/ydb/core/blobstorage/testload/test_load_write.cpp b/ydb/core/blobstorage/testload/test_load_write.cpp index 3169c1609ed..7bf0e3ff992 100644 --- a/ydb/core/blobstorage/testload/test_load_write.cpp +++ b/ydb/core/blobstorage/testload/test_load_write.cpp @@ -14,7 +14,7 @@ #include <util/datetime/cputimer.h> #include <util/generic/queue.h> #include <util/generic/set.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/random/fast.h> namespace NKikimr { diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp index ac7dcd7d58a..6d3f6a12adb 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_vmultiput_actor.cpp @@ -76,7 +76,7 @@ namespace NKikimr { cookie = &cookieValue; } - TInstant now = TAppData::TimeProvider->Now(); + TInstant now = TAppData::TimeProvider->Now(); const ui64 bufferSizeBytes = Event->Get()->GetBufferBytes(); auto vMultiPutResult = std::make_unique<TEvBlobStorage::TEvVMultiPutResult>(NKikimrProto::OK, vdisk, cookie, now, Event->Get()->GetCachedByteSize(), &vMultiPutRecord, SkeletonFrontIDPtr, MultiPutResMsgsPtr, diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_committer.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_committer.cpp index e02b627a698..45029c3a2ad 100644 --- a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_committer.cpp +++ b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_committer.cpp @@ -149,7 +149,7 @@ namespace NKikimr { CHECK_PDISK_RESPONSE(SlCtx->VCtx, ev, ctx); Y_VERIFY(ev->Get()->Results.size() == 1); const ui64 entryPointLsn = ev->Get()->Results[0].Lsn; - TCommitHistory commitHistory(TAppData::TimeProvider->Now(), entryPointLsn, EntryPointSerializer.RecoveryLogConfirmedLsn); + TCommitHistory commitHistory(TAppData::TimeProvider->Now(), entryPointLsn, EntryPointSerializer.RecoveryLogConfirmedLsn); ctx.Send(NotifyID, new TEvSyncLogCommitDone(commitHistory, EntryPointSerializer.GetEntryPointDbgInfo(), std::move(Delta))); Die(ctx); diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp index 237bba147b9..6810859d2d1 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp @@ -16,7 +16,7 @@ #include <util/generic/is_in.h> #include <util/string/join.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <list> #include <typeinfo> @@ -391,7 +391,7 @@ protected: } else { UNIT_ASSERT_C(IsIn(destroyedActors, actorId), "Test actor or its child wasn't destroyed. Actor id: " << actorId - << ". Type: " << TypeName(*actor)); + << ". Type: " << TypeName(*actor)); } } } diff --git a/ydb/core/erasure/erasure_rope.cpp b/ydb/core/erasure/erasure_rope.cpp index b1de833d936..10870800828 100644 --- a/ydb/core/erasure/erasure_rope.cpp +++ b/ydb/core/erasure/erasure_rope.cpp @@ -46,7 +46,7 @@ namespace NErasureRope { static void Refurbish(TRope &str, ui64 size) { if (str.GetSize() != size) { - str = TRopeHelpers::RopeUninitialized(size); + str = TRopeHelpers::RopeUninitialized(size); } } @@ -111,13 +111,13 @@ static const std::array<TErasureParameters, TRopeErasureType::ErasureSpeciesCoun ,{TRopeErasureType::ErasureMirror, 1, 2, 1} // 18 = ErasureSpicies::ErasureMirror3of4 }}; -void PadAndCrcAtTheEnd(TRopeHelpers::Iterator data, ui64 dataSize, ui64 bufferSize) { +void PadAndCrcAtTheEnd(TRopeHelpers::Iterator data, ui64 dataSize, ui64 bufferSize) { ui64 marginSize = bufferSize - dataSize - sizeof(ui32); if (marginSize) { - TRopeUtils::Memset(data + dataSize, 0, marginSize); + TRopeUtils::Memset(data + dataSize, 0, marginSize); } - ui32 hash = TRopeHelpers::GetCrc32c(data, dataSize); - TRopeUtils::Memcpy(data + (bufferSize - sizeof(ui32)), (const char *)&hash, sizeof(ui32)); + ui32 hash = TRopeHelpers::GetCrc32c(data, dataSize); + TRopeUtils::Memcpy(data + (bufferSize - sizeof(ui32)), (const char *)&hash, sizeof(ui32)); } bool CheckCrcAtTheEnd(TRopeErasureType::ECrcMode crcMode, const TRope& buf) { @@ -130,7 +130,7 @@ bool CheckCrcAtTheEnd(TRopeErasureType::ECrcMode crcMode, const TRope& buf) { } else { Y_VERIFY(buf.GetSize() >= sizeof(ui32), "Error in CheckWholeBlobCrc: blob part size# %" PRIu64 " is less then crcSize# %" PRIu64, (ui64)buf.GetSize(), (ui64)sizeof(ui32)); - ui32 crc = TRopeHelpers::GetCrc32c(buf.Begin(), buf.GetSize() - sizeof(ui32)); + ui32 crc = TRopeHelpers::GetCrc32c(buf.Begin(), buf.GetSize() - sizeof(ui32)); TString expectedStringCrc = TRope(buf.Begin() + buf.GetSize() - sizeof(ui32), buf.End()).ConvertToString(); ui32 expectedCrc = ReadUnaligned<ui32>(expectedStringCrc.data()); return crc == expectedCrc; @@ -168,9 +168,9 @@ public: ui32 Prime; TRopeErasureType::ECrcMode CrcMode; - using TBufferDataPart = TStackVec<TRopeHelpers::TRopeFastView, MAX_TOTAL_PARTS>; + using TBufferDataPart = TStackVec<TRopeHelpers::TRopeFastView, MAX_TOTAL_PARTS>; TBufferDataPart BufferDataPart; - TRopeHelpers::TRopeFastView Data; + TRopeHelpers::TRopeFastView Data; TBlockParams(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, ui64 dataSize) { DataSize = dataSize; @@ -201,14 +201,14 @@ public: CrcMode = crcMode; } - inline void PrepareInputDataPointers(TRopeHelpers::Iterator data) { + inline void PrepareInputDataPointers(TRopeHelpers::Iterator data) { BufferDataPart.resize(DataParts); for (ui32 i = 0; i < FirstSmallPartIdx; ++i) { - BufferDataPart[i] = TRopeHelpers::TRopeFastView(data); + BufferDataPart[i] = TRopeHelpers::TRopeFastView(data); data += LargePartSize; } for (ui32 i = FirstSmallPartIdx; i < DataParts; ++i) { - BufferDataPart[i] = TRopeHelpers::TRopeFastView(data); + BufferDataPart[i] = TRopeHelpers::TRopeFastView(data); data += SmallPartSize; } } @@ -244,7 +244,7 @@ public: // Use the remaining parts to fill in the last block // Write the tail of the data if (TailSize) { - TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; PrepareLastBlockData(lastBlockSource.Begin(), bufferDataPart); @@ -404,7 +404,7 @@ public: } else { char buffer[MAX_TOTAL_PARTS][(MAX_TOTAL_PARTS - 2) * sizeof(ui64)]; for (ui32 i = 0; i < DataParts; ++i) { - TRopeUtils::Memcpy(buffer[i], outPartSet.Parts[i].FastViewer.GetCurrent(writePosition), ColumnSize); + TRopeUtils::Memcpy(buffer[i], outPartSet.Parts[i].FastViewer.GetCurrent(writePosition), ColumnSize); } EoSplitLoop<false>(outPartSet, writePosition, 1u, buffer); } @@ -483,40 +483,40 @@ public: #undef IN_EL } - void PrepareLastBlockData(TRopeHelpers::Iterator lastBlockSource, TBufferDataPart &bufferDataPart) { + void PrepareLastBlockData(TRopeHelpers::Iterator lastBlockSource, TBufferDataPart &bufferDataPart) { bufferDataPart.resize(DataParts); for (ui32 i = 0; i < FirstSmallPartIdx; ++i) { - bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource); - TRopeUtils::Memcpy(bufferDataPart[i].GetBegin(), BufferDataPart[i].GetBegin() + WholeBlocks * ColumnSize, + bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource); + TRopeUtils::Memcpy(bufferDataPart[i].GetBegin(), BufferDataPart[i].GetBegin() + WholeBlocks * ColumnSize, ColumnSize); lastBlockSource += ColumnSize; } for (ui32 i = FirstSmallPartIdx; i < DataParts - 1; ++i) { - bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource); - TRopeUtils::Memset(bufferDataPart[i].GetBegin(), 0, ColumnSize); + bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource); + TRopeUtils::Memset(bufferDataPart[i].GetBegin(), 0, ColumnSize); lastBlockSource += ColumnSize; } - bufferDataPart[DataParts - 1] = TRopeHelpers::TRopeFastView(lastBlockSource); + bufferDataPart[DataParts - 1] = TRopeHelpers::TRopeFastView(lastBlockSource); if (LastPartTailSize) { - TRopeUtils::Memcpy(bufferDataPart[DataParts - 1].GetBegin(), BufferDataPart[DataParts - 1].GetBegin() + WholeBlocks * ColumnSize, + TRopeUtils::Memcpy(bufferDataPart[DataParts - 1].GetBegin(), BufferDataPart[DataParts - 1].GetBegin() + WholeBlocks * ColumnSize, LastPartTailSize); } - TRopeUtils::Memset(bufferDataPart[DataParts - 1].GetBegin() + LastPartTailSize, 0, ColumnSize - LastPartTailSize); + TRopeUtils::Memset(bufferDataPart[DataParts - 1].GetBegin() + LastPartTailSize, 0, ColumnSize - LastPartTailSize); } - void PrepareLastBlockPointers(TRopeHelpers::Iterator lastBlockSource, TBufferDataPart &bufferDataPart) { + void PrepareLastBlockPointers(TRopeHelpers::Iterator lastBlockSource, TBufferDataPart &bufferDataPart) { bufferDataPart.resize(DataParts); for (ui32 i = 0; i < DataParts; ++i) { - bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource); + bufferDataPart[i] = TRopeHelpers::TRopeFastView(lastBlockSource); lastBlockSource += ColumnSize; } } void PlaceLastBlock(TBufferDataPart& bufferDataPart) { for (ui32 i = 0; i < FirstSmallPartIdx; ++i) { - TRopeUtils::Memcpy(BufferDataPart[i].GetBegin() + WholeBlocks * ColumnSize, bufferDataPart[i].GetBegin(), ColumnSize); + TRopeUtils::Memcpy(BufferDataPart[i].GetBegin() + WholeBlocks * ColumnSize, bufferDataPart[i].GetBegin(), ColumnSize); } - TRopeUtils::Memcpy(BufferDataPart[DataParts - 1].GetBegin() + WholeBlocks * ColumnSize, + TRopeUtils::Memcpy(BufferDataPart[DataParts - 1].GetBegin() + WholeBlocks * ColumnSize, bufferDataPart[DataParts - 1].GetBegin(), LastPartTailSize); } @@ -528,7 +528,7 @@ public: // Use the remaining parts to fill in the last block // Write the tail of the data if (TailSize) { - TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; if (!isFromDataParts) { PrepareLastBlockData(lastBlockSource.Begin(), bufferDataPart); @@ -562,28 +562,28 @@ public: ui64 partSize = i < FirstSmallPartIdx ? LargePartSize : SmallPartSize; partSize = i == DataParts - 1 ? SmallPartSize + LastPartTailSize : partSize; if (!restoreParts && (i == missingPartIdxA || i == missingPartIdxB)) { - dst.Insert(dst.End(), TRopeHelpers::RopeUninitialized(partSize)); + dst.Insert(dst.End(), TRopeHelpers::RopeUninitialized(partSize)); } else { dst.Insert(dst.End(), TRope(part.OwnedRope.Begin(), part.OwnedRope.Begin() + partSize)); } } } - void GlueBlockPartsMemcpy(TRopeHelpers::Iterator dst, const TDataPartSet& partSet) const { + void GlueBlockPartsMemcpy(TRopeHelpers::Iterator dst, const TDataPartSet& partSet) const { if (LargePartSize) { for (ui32 i = 0; i < FirstSmallPartIdx; ++i) { - TRopeUtils::Memcpy(dst, partSet.Parts[i].OwnedRope.Begin(), LargePartSize); + TRopeUtils::Memcpy(dst, partSet.Parts[i].OwnedRope.Begin(), LargePartSize); dst += LargePartSize; } if (SmallPartSize) { for (ui32 i = FirstSmallPartIdx; i < DataParts - 1; ++i) { - TRopeUtils::Memcpy(dst, partSet.Parts[i].OwnedRope.Begin(), SmallPartSize); + TRopeUtils::Memcpy(dst, partSet.Parts[i].OwnedRope.Begin(), SmallPartSize); dst += SmallPartSize; } } } if (SmallPartSize + LastPartTailSize) { - TRopeUtils::Memcpy(dst, partSet.Parts[DataParts - 1].OwnedRope.Begin(), SmallPartSize + LastPartTailSize); + TRopeUtils::Memcpy(dst, partSet.Parts[DataParts - 1].OwnedRope.Begin(), SmallPartSize + LastPartTailSize); } return; } @@ -731,16 +731,16 @@ public: for (ui32 i = 0; i < DataParts; ++i) { if (i != missingDataPartIdx) { - TRopeUtils::Memcpy(partsBuffer[i], partSet.Parts[i].FastViewer.GetCurrent(readPosition), ColumnSize); + TRopeUtils::Memcpy(partsBuffer[i], partSet.Parts[i].FastViewer.GetCurrent(readPosition), ColumnSize); } } if (restoreParts) { - TRopeUtils::Memcpy(partsBuffer[missingDataPartIdx], + TRopeUtils::Memcpy(partsBuffer[missingDataPartIdx], partSet.Parts[missingDataPartIdx].FastViewer.GetCurrent(readPosition), ColumnSize); } - TRopeUtils::Memcpy(partsBuffer[lastColumn], + TRopeUtils::Memcpy(partsBuffer[lastColumn], partSet.Parts[lastColumn].FastViewer.GetCurrent(readPosition), ColumnSize); VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12); @@ -814,13 +814,13 @@ public: VERBOSE_COUT_BLOCK(restoreFullData, OUT_EL_BLOCK, IN_EL, IN_M, IN_M12); if (restoreParts) { - TRopeUtils::Memcpy(partSet.Parts[missingDataPartIdx].FastViewer.GetCurrent(readPosition), + TRopeUtils::Memcpy(partSet.Parts[missingDataPartIdx].FastViewer.GetCurrent(readPosition), partsBuffer[missingDataPartIdx], ColumnSize); } if (restoreFullData) { for (ui32 i = 0; i < DataParts; ++i) { - TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent(blockIdx * LineCount * sizeof(ui64)), + TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent(blockIdx * LineCount * sizeof(ui64)), (const char*)fullDataBuffer[i], LineCount * sizeof(ui64)); } } @@ -858,7 +858,7 @@ public: // Read the tail of the data if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) { TRACE("EoDiagonalRestorePart tail" << Endl); - TRope lastBlock = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlock = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; PrepareLastBlockPointers(lastBlock.Begin(), bufferDataPart); @@ -1242,12 +1242,12 @@ public: for (ui32 i = 0; i <= DataParts; ++i) { if (i != missingDataPartIdxA && i != missingDataPartIdxB) { - TRopeUtils::Memcpy(partsBuffer[i], + TRopeUtils::Memcpy(partsBuffer[i], partSet.Parts[i].FastViewer.GetCurrent(readPosition), ColumnSize); } } - TRopeUtils::Memcpy(partsBuffer[lastColumn], + TRopeUtils::Memcpy(partsBuffer[lastColumn], partSet.Parts[lastColumn].FastViewer.GetCurrent(readPosition), ColumnSize); VERBOSE_COUT_BLOCK(true, IN_EL, IN_EL, IN_M, IN_M12); @@ -1372,7 +1372,7 @@ public: if (restoreFullData) { for (ui32 i = 0; i < DataParts; ++i) { - TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent(blockIdx * LineCount * sizeof(ui64)), + TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent(blockIdx * LineCount * sizeof(ui64)), (const char *) fullDataBuffer[i], LineCount * sizeof(ui64)); } } @@ -1390,7 +1390,7 @@ public: partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB); if (TailSize) { - TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart); @@ -1422,7 +1422,7 @@ public: partSet, 0ull, WholeBlocks, missingDataPartIdxA, missingDataPartIdxB, missingDataPartIdxC); if (TailSize) { - TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart); @@ -1468,7 +1468,7 @@ public: if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) { TRACE("EoMainRestoreParts restore tail" << Endl); - TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart); @@ -1575,7 +1575,7 @@ public: for (ui32 i = 0; i <= DataParts; ++i) { if (i != missingDataPartIdx) { - TRopeUtils::Memcpy((char*)partsBuffer[i], + TRopeUtils::Memcpy((char*)partsBuffer[i], partSet.Parts[i].FastViewer.GetCurrent(readPosition), LineCount * sizeof(ui64)); } } @@ -1612,7 +1612,7 @@ public: if (restoreFullData) { for (ui32 i = 0; i < DataParts; ++i) { - TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent((writePosition - LineCount) * sizeof(ui64)), + TRopeUtils::Memcpy(bufferDataPart[i].GetCurrent((writePosition - LineCount) * sizeof(ui64)), (const char*)fullDataBuffer[i], LineCount * sizeof(ui64)); } } @@ -1649,7 +1649,7 @@ public: if (TailSize && (partSet.Parts[presentPartIdx].Size + readPosition > WholeBlocks * ColumnSize)) { TRACE("Restore tail, restoreFullData# " << restoreFullData << " resotreParts# " << restoreParts << Endl); - TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); + TRope lastBlockSource = TRopeHelpers::CreateRope(MAX_TOTAL_PARTS * (MAX_TOTAL_PARTS - 2) * sizeof(ui64)); TBufferDataPart bufferDataPart; PrepareLastBlockPointers(lastBlockSource.Begin(), bufferDataPart); @@ -1679,7 +1679,7 @@ public: if (DataSize) { PadAndCrcAtTheEnd(inOutPartSet.Parts[partIdx].OwnedRope.Begin(), PartUserSize, PartContainerSize); } else { - TRopeUtils::Memset(inOutPartSet.Parts[partIdx].OwnedRope.Begin(), 0, PartContainerSize); + TRopeUtils::Memset(inOutPartSet.Parts[partIdx].OwnedRope.Begin(), 0, PartContainerSize); } return; } @@ -1701,7 +1701,7 @@ void PadAndCrcParts(TRopeErasureType::ECrcMode crcMode, const TBlockParams &p, T } } else { for (ui32 i = 0; i < p.TotalParts; ++i) { - TRopeUtils::Memset(inOutPartSet.Parts[i].OwnedRope.Begin(), 0, p.PartContainerSize); + TRopeUtils::Memset(inOutPartSet.Parts[i].OwnedRope.Begin(), 0, p.PartContainerSize); } } return; @@ -1711,7 +1711,7 @@ void PadAndCrcParts(TRopeErasureType::ECrcMode crcMode, const TBlockParams &p, T inline void StarBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope &buffer, TDataPartSet &outPartSet) { - Y_VERIFY(TRopeHelpers::Is8Aligned(buffer)); + Y_VERIFY(TRopeHelpers::Is8Aligned(buffer)); TBlockParams p(crcMode, type, buffer.GetSize()); // Prepare input data pointers @@ -1753,7 +1753,7 @@ inline void EoBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureT rope.Insert(rope.End(), TRope(iterator, buffer.End())); } - TRopeHelpers::Resize(rope, p.PartContainerSize); + TRopeHelpers::Resize(rope, p.PartContainerSize); outPartSet.Parts[i].ReferenceTo(rope); } @@ -1771,7 +1771,7 @@ inline void EoBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureT inline void XorBlockSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &type, const TRope& buffer, TDataPartSet& outPartSet) { - Y_VERIFY(TRopeHelpers::Is8Aligned(buffer)); + Y_VERIFY(TRopeHelpers::Is8Aligned(buffer)); TBlockParams p(crcMode, type, buffer.GetSize()); // Prepare input data pointers @@ -2577,7 +2577,7 @@ void MirrorSplit(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &typ ui64 partSize = type.PartSize(crcMode, buffer.GetSize()); TRope& part = outPartSet.FullDataFragment.OwnedRope; part = buffer; - TRopeHelpers::Resize(part, partSize); + TRopeHelpers::Resize(part, partSize); if (buffer.GetSize() || part.GetSize()) { Y_VERIFY(part.GetSize() >= buffer.GetSize() + sizeof(ui32), "Part size too small, buffer size# %" PRIu64 " partSize# %" PRIu64, (ui64)buffer.GetSize(), (ui64)partSize); @@ -2617,7 +2617,7 @@ void MirrorRestore(TRopeErasureType::ECrcMode crcMode, const TRopeErasureType &t TRope outBuffer = partSet.Parts[partIdx].OwnedRope; Y_VERIFY(outBuffer.GetSize() >= partSet.FullDataSize, "Unexpected outBuffer.size# %" PRIu64 " fullDataSize# %" PRIu64, (ui64)outBuffer.GetSize(), (ui64)partSet.FullDataSize); - TRopeHelpers::Resize(outBuffer, partSet.FullDataSize); // To pad with zeroes! + TRopeHelpers::Resize(outBuffer, partSet.FullDataSize); // To pad with zeroes! partSet.FullDataFragment.ReferenceTo(outBuffer); return; } diff --git a/ydb/core/erasure/erasure_rope.h b/ydb/core/erasure/erasure_rope.h index a4835ce8341..16383ec1382 100644 --- a/ydb/core/erasure/erasure_rope.h +++ b/ydb/core/erasure/erasure_rope.h @@ -17,7 +17,7 @@ namespace NKikimr { namespace NErasureRope { -class TRopeHelpers { +class TRopeHelpers { public: using Iterator = TRope::TConstIterator; @@ -51,13 +51,13 @@ public: static TRope RopeCopy(const TRope& src) { TRope copy = RopeUninitialized(src.GetSize()); - TRopeUtils::Memcpy(copy.Begin(), src.Begin(), src.GetSize()); + TRopeUtils::Memcpy(copy.Begin(), src.Begin(), src.GetSize()); return copy; } static TRope RopeFromStringMemcpy(const TString& string) { TRope rope = RopeUninitialized(string.size()); - TRopeUtils::Memcpy(rope.Begin(), string.data(), string.size()); + TRopeUtils::Memcpy(rope.Begin(), string.data(), string.size()); return rope; } @@ -235,7 +235,7 @@ struct TPartFragment { ui64 Size = 0; ui64 PartSize = 0; // Full size of the part - mutable TRopeHelpers::TRopeFastView FastViewer; + mutable TRopeHelpers::TRopeFastView FastViewer; TPartFragment() = default; @@ -244,7 +244,7 @@ struct TPartFragment { Offset = lhs.Offset; Size = lhs.Size; PartSize = lhs.PartSize; - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); FastViewer.SetOffset(Offset); } @@ -256,7 +256,7 @@ struct TPartFragment { Offset = lhs.Offset; Size = lhs.Size; PartSize = lhs.PartSize; - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); FastViewer.SetOffset(Offset); return *this; } @@ -270,20 +270,20 @@ struct TPartFragment { Offset = 0; Size = 0; PartSize = 0; - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); } void UninitializedOwnedWhole(ui64 size) { - OwnedRope = TRopeHelpers::RopeUninitialized(size); - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + OwnedRope = TRopeHelpers::RopeUninitialized(size); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); Offset = 0; Size = size; PartSize = size; } void ResetToWhole(const TRope& whole) { - OwnedRope = TRopeHelpers::RopeCopy(whole); - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + OwnedRope = TRopeHelpers::RopeCopy(whole); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); Offset = 0; Size = whole.GetSize(); PartSize = Size; @@ -291,7 +291,7 @@ struct TPartFragment { void ReferenceTo(const TRope& whole) { OwnedRope = whole; - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); Offset = 0; Size = whole.GetSize(); PartSize = Size; @@ -299,7 +299,7 @@ struct TPartFragment { void ReferenceTo(const TRope &piece, ui64 offset, ui64 size, ui64 partSize) { OwnedRope = piece; - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); Offset = offset; Y_VERIFY(size <= piece.GetSize()); Size = size; @@ -329,8 +329,8 @@ struct TPartFragment { } void ResetWithFullCopy() { - OwnedRope = TRopeHelpers::RopeCopy(OwnedRope); - FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); + OwnedRope = TRopeHelpers::RopeCopy(OwnedRope); + FastViewer = TRopeHelpers::TRopeFastView(OwnedRope); } ui64 GetContiguousSize(ui64 pos) const { @@ -359,7 +359,7 @@ struct TDataPartSet { bool Is8Aligned() const { for (ui32 i = 0; i < 8u; ++i) { - if ((PartsMask & (1u << i)) && !TRopeHelpers::Is8Aligned(Parts[i].OwnedRope)) { + if ((PartsMask & (1u << i)) && !TRopeHelpers::Is8Aligned(Parts[i].OwnedRope)) { return false; } } diff --git a/ydb/core/erasure/erasure_rope_ut.cpp b/ydb/core/erasure/erasure_rope_ut.cpp index 62a6242ea92..e9ccae016c9 100644 --- a/ydb/core/erasure/erasure_rope_ut.cpp +++ b/ydb/core/erasure/erasure_rope_ut.cpp @@ -184,7 +184,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { testString[i] = (char)data[i]; } TDataPartSet partSet; - type.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); + type.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); for (ui32 i = 0; i < 6; ++i) { UNIT_ASSERT_EQUAL_C(partSet.Parts[i].size(), expectedParts[i].size(), Sprintf("%lu == %lu", partSet.Parts[i].size(), expectedParts[i].size())); @@ -251,7 +251,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { } TDataPartSet partSet; try { - groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); + groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); } catch (yexception ex) { ex << " [in SplitData while testing " << errorInfo << "]"; throw ex; @@ -380,7 +380,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { // TRope rope; // for (size_t i = 0; i < dataSize; ++i) { // TString base = GenerateRandomString(1); -// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); +// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); // } // Y_VERIFY(rope.GetSize() == dataSize); // return rope; @@ -393,11 +393,11 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { // TRope rope; // for (size_t i = 1; i < dataSize && rope.GetSize() + i <= dataSize; ++i) { // TString base = GenerateRandomString(i); -// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); +// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); // } // if (rope.GetSize() < dataSize) { // TString base = GenerateRandomString(dataSize - rope.GetSize()); -// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); +// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); // } // Y_VERIFY(rope.GetSize() == dataSize); // return rope; @@ -415,10 +415,10 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { // ui64 size = (ui64) randGen.GenRand() % maxSize; // if (rope.GetSize() + size <= dataSize) { // TString base = GenerateRandomString(size); -// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); +// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); // } else { // TString base = GenerateRandomString(dataSize - rope.GetSize()); -// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); +// rope.Insert(rope.End(), TRopeHelpers::RopeFromStringReference(base)); // } // } // Y_VERIFY(rope.GetSize() == dataSize); @@ -509,7 +509,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { ((char*)testString.data())[i] = (char)(i % 10) + '0'; } - inputBuffer = TRopeHelpers::RopeFromStringMemcpy(testString); + inputBuffer = TRopeHelpers::RopeFromStringMemcpy(testString); } TDataPartSet partSet; @@ -759,7 +759,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { ((ui8*)testString.data())[i] = (ui8)i; } TDataPartSet partSet; - groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); + groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); for (ui32 p = 0; p < groupType.DataParts(); ++p) { auto &part = partSet.Parts[p]; VERBOSE_COUT("Part# " << p << " Size# " << part.size() << " Data# "); @@ -823,7 +823,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { // Split the data into parts TDataPartSet partSet; - groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); + groupType.SplitData(TRopeErasureType::CrcModeNone, TRopeHelpers::RopeFromStringMemcpy(testString), partSet); ui64 partSize = groupType.PartSize(TRopeErasureType::CrcModeNone, dataSize); for (ui32 part = 0; part < groupType.TotalPartCount(); ++part) { UNIT_ASSERT_EQUAL(partSize, partSet.Parts[part].size()); @@ -887,7 +887,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { for (ui32 idx = 0; idx < partSet.Parts.size(); ++idx) { ui32 cutBegin = Min(partSize, needBegin); ui32 cutSize = Min(partSize, needEnd) - cutBegin; - partSet.Parts[idx].ReferenceTo(TRopeHelpers::RopeFromStringMemcpy( + partSet.Parts[idx].ReferenceTo(TRopeHelpers::RopeFromStringMemcpy( partSet.Parts[idx].OwnedRope.ConvertToString().substr( cutBegin, cutSize)), cutBegin, cutSize, partSize); } @@ -900,7 +900,7 @@ Y_UNIT_TEST_SUITE(TErasureTypeTest) { ui32 cutSize = Min(partSize, needEnd) - cutBegin; partSet.Parts[idx].clear(); TString tmp = TString::Uninitialized(cutSize); - partSet.Parts[idx].ReferenceTo(TRopeHelpers::RopeFromStringMemcpy(tmp), cutBegin, cutSize, partSize); + partSet.Parts[idx].ReferenceTo(TRopeHelpers::RopeFromStringMemcpy(tmp), cutBegin, cutSize, partSize); } // Restore the data diff --git a/ydb/core/grpc_services/counters/counters.cpp b/ydb/core/grpc_services/counters/counters.cpp index 8fc8ede9174..9c862f3dab8 100644 --- a/ydb/core/grpc_services/counters/counters.cpp +++ b/ydb/core/grpc_services/counters/counters.cpp @@ -431,7 +431,7 @@ public: return dbCounters->GetCounterBlock(serviceName, requestName); } - dbCounters = DbCounters.InsertIfAbsentWithInit(database, [&database, this] { + dbCounters = DbCounters.InsertIfAbsentWithInit(database, [&database, this] { auto counters = MakeIntrusive<TGRpcDbCounters>(); if (ActorSystem) { diff --git a/ydb/core/keyvalue/keyvalue_collect_operation.h b/ydb/core/keyvalue/keyvalue_collect_operation.h index 49c67c34cf7..93647399343 100644 --- a/ydb/core/keyvalue/keyvalue_collect_operation.h +++ b/ydb/core/keyvalue/keyvalue_collect_operation.h @@ -52,10 +52,10 @@ struct TCollectOperation : public TThrRefBase { TVector<TLogoBlobID> DoNotKeep; TCollectOperation(ui64 collectGeneration, ui64 collectStep, - TVector<TLogoBlobID> &&keep, TVector<TLogoBlobID> &&doNotKeep) + TVector<TLogoBlobID> &&keep, TVector<TLogoBlobID> &&doNotKeep) : Header(collectGeneration, collectStep, keep, doNotKeep) - , Keep(std::move(keep)) - , DoNotKeep(std::move(doNotKeep)) + , Keep(std::move(keep)) + , DoNotKeep(std::move(doNotKeep)) {} }; diff --git a/ydb/core/keyvalue/keyvalue_collector_ut.cpp b/ydb/core/keyvalue/keyvalue_collector_ut.cpp index 4456a823803..22066f45b2e 100644 --- a/ydb/core/keyvalue/keyvalue_collector_ut.cpp +++ b/ydb/core/keyvalue/keyvalue_collector_ut.cpp @@ -113,7 +113,7 @@ Y_UNIT_TEST(TestKeyValueCollectorEmpty) { TVector<TLogoBlobID> keep; TVector<TLogoBlobID> doNotKeep; - TIntrusivePtr<NKeyValue::TCollectOperation> operation(new NKeyValue::TCollectOperation(100, 100, std::move(keep), std::move(doNotKeep))); + TIntrusivePtr<NKeyValue::TCollectOperation> operation(new NKeyValue::TCollectOperation(100, 100, std::move(keep), std::move(doNotKeep))); context.SetActor(CreateKeyValueCollector( context.GetTabletActorId(), operation, context.GetTabletInfo().Get(), 200, 200, true)); @@ -140,7 +140,7 @@ Y_UNIT_TEST(TestKeyValueCollectorSingle) { TVector<TLogoBlobID> keep; keep.emplace_back(0x10010000001000Bull, 5, 58949, NKeyValue::BLOB_CHANNEL, 1209816, 10); TVector<TLogoBlobID> doNotKeep; - TIntrusivePtr<NKeyValue::TCollectOperation> operation(new NKeyValue::TCollectOperation(100, 100, std::move(keep), std::move(doNotKeep))); + TIntrusivePtr<NKeyValue::TCollectOperation> operation(new NKeyValue::TCollectOperation(100, 100, std::move(keep), std::move(doNotKeep))); context.SetActor(CreateKeyValueCollector( context.GetTabletActorId(), operation, context.GetTabletInfo().Get(), 200, 200, true)); @@ -190,7 +190,7 @@ Y_UNIT_TEST(TestKeyValueCollectorMultiple) { ids.insert(doNotKeep[i]); } - TIntrusivePtr<NKeyValue::TCollectOperation> operation(new NKeyValue::TCollectOperation(100, 100, std::move(keep), std::move(doNotKeep))); + TIntrusivePtr<NKeyValue::TCollectOperation> operation(new NKeyValue::TCollectOperation(100, 100, std::move(keep), std::move(doNotKeep))); context.SetActor(CreateKeyValueCollector( context.GetTabletActorId(), operation, context.GetTabletInfo().Get(), 200, 200, true)); diff --git a/ydb/core/keyvalue/keyvalue_flat_impl.h b/ydb/core/keyvalue/keyvalue_flat_impl.h index 292f2feff93..b4e2123ec4c 100644 --- a/ydb/core/keyvalue/keyvalue_flat_impl.h +++ b/ydb/core/keyvalue/keyvalue_flat_impl.h @@ -365,13 +365,13 @@ protected: State.OnRequestComplete(event.RequestUid, event.Generation, event.Step, ctx, Info(), event.Status, event.Stat); } - void Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr &ev, const TActorContext &ctx) { - Y_UNUSED(ev); - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletID() - << " Handle TEvCollectGarbageResult Marker# KV52"); - State.RegisterInitialCollectResult(ctx); - } - + void Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr &ev, const TActorContext &ctx) { + Y_UNUSED(ev); + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletID() + << " Handle TEvCollectGarbageResult Marker# KV52"); + State.RegisterInitialCollectResult(ctx); + } + void Handle(TEvKeyValue::TEvRequest::TPtr ev, const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletID() << " Handle TEvRequest " << ev->Get()->ToString()); @@ -483,7 +483,7 @@ public: HFunc(TEvKeyValue::TEvNotify, Handle); HFunc(TEvKeyValue::TEvPeriodicRefresh, Handle); HFunc(TChannelBalancer::TEvUpdateWeights, Handle); - HFunc(TEvBlobStorage::TEvCollectGarbageResult, Handle); + HFunc(TEvBlobStorage::TEvCollectGarbageResult, Handle); HFunc(TEvents::TEvPoisonPill, Handle); default: diff --git a/ydb/core/keyvalue/keyvalue_helpers.cpp b/ydb/core/keyvalue/keyvalue_helpers.cpp index cd4051d8cd3..6f8cffd599a 100644 --- a/ydb/core/keyvalue/keyvalue_helpers.cpp +++ b/ydb/core/keyvalue/keyvalue_helpers.cpp @@ -111,9 +111,9 @@ void THelpers::DbEraseCollect(ISimpleDb &db, const TActorContext &ctx) { db.Erase(key, ctx); } -THelpers::TGenerationStep THelpers::GenerationStep(const TLogoBlobID &id) { - return std::make_tuple(id.Generation(), id.Step()); -} - +THelpers::TGenerationStep THelpers::GenerationStep(const TLogoBlobID &id) { + return std::make_tuple(id.Generation(), id.Step()); +} + } // NKeyValue } // NKikimr diff --git a/ydb/core/keyvalue/keyvalue_helpers.h b/ydb/core/keyvalue/keyvalue_helpers.h index 68960f3b868..ccfda2982f0 100644 --- a/ydb/core/keyvalue/keyvalue_helpers.h +++ b/ydb/core/keyvalue/keyvalue_helpers.h @@ -27,9 +27,9 @@ struct THelpers { static void DbEraseCollect(ISimpleDb &db, const TActorContext &ctx); static void DbUpdateCollect(ui64 collectGeneration, ui64 collectStep, TVector<TLogoBlobID> keep, TVector<TLogoBlobID> doNotKeep, ISimpleDb &db, const TActorContext &ctx); - - using TGenerationStep = std::tuple<ui32, ui32>; - static TGenerationStep GenerationStep(const TLogoBlobID &id); + + using TGenerationStep = std::tuple<ui32, ui32>; + static TGenerationStep GenerationStep(const TLogoBlobID &id); }; } // NKeyValue diff --git a/ydb/core/keyvalue/keyvalue_state.cpp b/ydb/core/keyvalue/keyvalue_state.cpp index 6b31c463f83..18371d6a04c 100644 --- a/ydb/core/keyvalue/keyvalue_state.cpp +++ b/ydb/core/keyvalue/keyvalue_state.cpp @@ -421,10 +421,10 @@ void TKeyValueState::Load(const TString &key, const TString& value) { if (!item.IsInline()) { const ui32 newRefCount = ++RefCounts[item.LogoBlobId]; if (newRefCount == 1) { - CountWriteRecord(item.LogoBlobId.Channel(), item.LogoBlobId.BlobSize()); + CountWriteRecord(item.LogoBlobId.Channel(), item.LogoBlobId.BlobSize()); } - } else { - CountWriteRecord(0, item.InlineData.size()); + } else { + CountWriteRecord(0, item.InlineData.size()); } } break; @@ -459,7 +459,7 @@ void TKeyValueState::Load(const TString &key, const TString& value) { memcpy((char *) &doNotKeep[0], data, sizeof(TLogoBlobID) * doNotKeep.size()); } CollectOperation.Reset(new TCollectOperation( - header->CollectGeneration, header->CollectStep, std::move(keep), std::move(doNotKeep))); + header->CollectGeneration, header->CollectStep, std::move(keep), std::move(doNotKeep))); break; } case EIT_STATE: { @@ -504,7 +504,7 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e // Issue hard barriers { using TGroupChannel = std::tuple<ui32, ui8>; - THashMap<TGroupChannel, THelpers::TGenerationStep> hardBarriers; + THashMap<TGroupChannel, THelpers::TGenerationStep> hardBarriers; for (const auto &kv : RefCounts) { // extract blob id and validate its channel const TLogoBlobID &id = kv.first; @@ -515,11 +515,11 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e // are always inclusive const ui32 generation = id.Generation(); const ui32 step = id.Step(); - TMaybe<THelpers::TGenerationStep> current; + TMaybe<THelpers::TGenerationStep> current; if (step) { - current = THelpers::TGenerationStep(generation, step - 1); + current = THelpers::TGenerationStep(generation, step - 1); } else if (generation) { - current = THelpers::TGenerationStep(generation - 1, Max<ui32>()); + current = THelpers::TGenerationStep(generation - 1, Max<ui32>()); } // update minimum barrier value for this channel/group @@ -543,7 +543,7 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e for (const auto &history : channel.History) { const TGroupChannel key(history.GroupID, channel.Channel); if (!hardBarriers.count(key)) { - hardBarriers.emplace(key, THelpers::TGenerationStep(executorGeneration - 1, Max<ui32>())); + hardBarriers.emplace(key, THelpers::TGenerationStep(executorGeneration - 1, Max<ui32>())); } } } @@ -580,8 +580,8 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e const TLogoBlobID &id = refInfo.first; Y_VERIFY(id.Channel() >= BLOB_CHANNEL); - const THelpers::TGenerationStep blobGenStep = THelpers::GenerationStep(id); - const THelpers::TGenerationStep storedGenStep(StoredState.GetCollectGeneration(), StoredState.GetCollectStep()); + const THelpers::TGenerationStep blobGenStep = THelpers::GenerationStep(id); + const THelpers::TGenerationStep storedGenStep(StoredState.GetCollectGeneration(), StoredState.GetCollectStep()); // Mark with keep flag only new blobs if (storedGenStep < blobGenStep) { const ui32 group = info->GroupFor(id.Channel(), id.Generation()); @@ -613,7 +613,7 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e } } } - InitialCollectsSent = keepForGroupChannel.size(); + InitialCollectsSent = keepForGroupChannel.size(); for (auto &keepInfo : keepForGroupChannel) { ui32 group; ui8 channel; @@ -627,7 +627,7 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e const TActorId nodeWarden = MakeBlobStorageNodeWardenID(ctx.SelfID.NodeId()); const TActorId proxy = MakeBlobStorageProxyID(group); - ctx.ExecutorThread.Send(new IEventHandle(proxy, KeyValueActorId, ev.Release(), + ctx.ExecutorThread.Send(new IEventHandle(proxy, KeyValueActorId, ev.Release(), IEventHandle::FlagForwardOnNondelivery, 0, &nodeWarden)); } } @@ -635,18 +635,18 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e // Make a copy of the channel history (need a way to map channel/generation to history record) // History entry contains data on from-to generation for each channel auto &channels = info->Channels; - ChannelRangeSets.reserve(channels.size()); + ChannelRangeSets.reserve(channels.size()); for (ui64 idx = 0; idx < channels.size(); ++idx) { auto &channelInfo = channels[idx]; if (channelInfo.Channel < BLOB_CHANNEL) { // Remove (do not add) non-KV tablet managed channels continue; } - if (channelInfo.Channel >= ChannelRangeSets.size()) { - ChannelRangeSets.resize(channelInfo.Channel + 1); + if (channelInfo.Channel >= ChannelRangeSets.size()) { + ChannelRangeSets.resize(channelInfo.Channel + 1); } auto &history = channelInfo.History; - auto &rangeSet = ChannelRangeSets[channelInfo.Channel]; + auto &rangeSet = ChannelRangeSets[channelInfo.Channel]; // Remove (do not add) the latest entries for all channels auto endIt = history.end(); if (endIt != history.begin()) { @@ -665,30 +665,30 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e auto &chain = recordIt->second.Chain; for (auto itemIt = chain.begin(); itemIt != chain.end(); ++itemIt) { TLogoBlobID &id = itemIt->LogoBlobId; - if (id.Channel() < ChannelRangeSets.size()) { - ChannelRangeSets[id.Channel()].Remove(id.Generation()); + if (id.Channel() < ChannelRangeSets.size()) { + ChannelRangeSets[id.Channel()].Remove(id.Generation()); } } } for (auto it = Trash.begin(); it != Trash.end(); ++it) { const TLogoBlobID &id = *it; - if (id.Channel() < ChannelRangeSets.size()) { - ChannelRangeSets[id.Channel()].Remove(id.Generation()); + if (id.Channel() < ChannelRangeSets.size()) { + ChannelRangeSets[id.Channel()].Remove(id.Generation()); } } if (CollectOperation.Get()) { auto &keep = CollectOperation->Keep; for (auto it = keep.begin(); it != keep.end(); ++it) { const TLogoBlobID &id = *it; - if (id.Channel() < ChannelRangeSets.size()) { - ChannelRangeSets[id.Channel()].Remove(id.Generation()); + if (id.Channel() < ChannelRangeSets.size()) { + ChannelRangeSets[id.Channel()].Remove(id.Generation()); } } auto &doNotKeep = CollectOperation->DoNotKeep; for (auto it = doNotKeep.begin(); it != doNotKeep.end(); ++it) { const TLogoBlobID &id = *it; - if (id.Channel() < ChannelRangeSets.size()) { - ChannelRangeSets[id.Channel()].Remove(id.Generation()); + if (id.Channel() < ChannelRangeSets.size()) { + ChannelRangeSets[id.Channel()].Remove(id.Generation()); } } // Patch collect operation generation and step @@ -696,36 +696,36 @@ void TKeyValueState::InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 e CollectOperation->Header.SetCollectGeneration(ExecutorGeneration); CollectOperation->Header.SetCollectStep(0); } - THelpers::DbUpdateState(StoredState, db, ctx); - - // corner case, if no CollectGarbage events were sent - if (InitialCollectsSent == 0) { - SendCutHistory(ctx); - } - if (CollectOperation.Get()) { - // finish collect operation from local base - IsCollectEventSent = true; - StoreCollectComplete(ctx); - } else { - // initiate collection if trash was loaded from local base - PrepareCollectIfNeeded(ctx); - } -} - -void TKeyValueState::RegisterInitialCollectResult(const TActorContext &ctx) { - LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " InitialCollectsSent# " << InitialCollectsSent << " Marker# KV50"); - if (--InitialCollectsSent == 0) { - SendCutHistory(ctx); - } -} - -void TKeyValueState::SendCutHistory(const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId - << " SendCutHistory Marker# KV51"); + THelpers::DbUpdateState(StoredState, db, ctx); + + // corner case, if no CollectGarbage events were sent + if (InitialCollectsSent == 0) { + SendCutHistory(ctx); + } + if (CollectOperation.Get()) { + // finish collect operation from local base + IsCollectEventSent = true; + StoreCollectComplete(ctx); + } else { + // initiate collection if trash was loaded from local base + PrepareCollectIfNeeded(ctx); + } +} + +void TKeyValueState::RegisterInitialCollectResult(const TActorContext &ctx) { + LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " InitialCollectsSent# " << InitialCollectsSent << " Marker# KV50"); + if (--InitialCollectsSent == 0) { + SendCutHistory(ctx); + } +} + +void TKeyValueState::SendCutHistory(const TActorContext &ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId + << " SendCutHistory Marker# KV51"); // Prepare and send a messages to the Local - for (ui64 channel = 0; channel < ChannelRangeSets.size(); ++channel) { - auto endForBegin = ChannelRangeSets[channel].EndForBegin; + for (ui64 channel = 0; channel < ChannelRangeSets.size(); ++channel) { + auto endForBegin = ChannelRangeSets[channel].EndForBegin; for (auto it = endForBegin.begin(); it != endForBegin.end(); ++it) { ui32 fromGeneration = it->first; TAutoPtr<TEvTablet::TEvCutTabletHistory> ev(new TEvTablet::TEvCutTabletHistory); @@ -737,7 +737,7 @@ void TKeyValueState::SendCutHistory(const TActorContext &ctx) { ctx.Send(localActorId, ev.Release()); } } - ChannelRangeSets.clear(); + ChannelRangeSets.clear(); } void TKeyValueState::OnInitQueueEmpty(const TActorContext &ctx) { @@ -772,8 +772,8 @@ TLogoBlobID TKeyValueState::AllocateLogoBlobId(ui32 size, ui32 storageChannelIdx } else { Step(); } - Y_VERIFY(!CollectOperation || THelpers::GenerationStep(id) > - THelpers::TGenerationStep(CollectOperation->Header.GetCollectGeneration(), CollectOperation->Header.GetCollectStep())); + Y_VERIFY(!CollectOperation || THelpers::GenerationStep(id) > + THelpers::TGenerationStep(CollectOperation->Header.GetCollectGeneration(), CollectOperation->Header.GetCollectStep())); return id; } @@ -1299,7 +1299,7 @@ void TKeyValueState::CmdTrimLeakedBlobs(THolder<TIntermediate>& intermediate, IS if (numItems < intermediate->TrimLeakedBlobs->MaxItemsToTrim) { LOG_WARN_S(ctx, NKikimrServices::KEYVALUE, "KeyValue# " << TabletId << " trimming " << id.ToString()); Trash.insert(id); - CountTrashRecord(id.BlobSize()); + CountTrashRecord(id.BlobSize()); THelpers::DbUpdateTrash(id, db, ctx); ++numItems; } else { @@ -1708,7 +1708,7 @@ void TKeyValueState::OnRequestComplete(ui64 requestUid, ui64 generation, ui64 st InFlightForStep.erase(it); // Initiate Garbage collection process if needed - StartCollectingIfPossible(ctx); + StartCollectingIfPossible(ctx); } } } @@ -1716,15 +1716,15 @@ void TKeyValueState::OnRequestComplete(ui64 requestUid, ui64 generation, ui64 st bool TKeyValueState::CheckDeadline(const TActorContext &ctx, NKikimrClient::TKeyValueRequest &kvRequest, THolder<TIntermediate> &intermediate) { if (kvRequest.HasDeadlineInstantMs()) { - TInstant now = TAppData::TimeProvider->Now(); + TInstant now = TAppData::TimeProvider->Now(); intermediate->Deadline = TInstant::MicroSeconds(kvRequest.GetDeadlineInstantMs() * 1000ull); - if (intermediate->Deadline <= now) { + if (intermediate->Deadline <= now) { TStringStream str; str << "KeyValue# " << TabletId; str << " Deadline reached before processing the request!"; str << " DeadlineInstantMs# " << (ui64)kvRequest.GetDeadlineInstantMs(); - str << " < Now# " << (ui64)now.MilliSeconds(); + str << " < Now# " << (ui64)now.MilliSeconds(); ReplyError(ctx, str.Str(), NMsgBusProxy::MSTATUS_TIMEOUT, intermediate); return true; } diff --git a/ydb/core/keyvalue/keyvalue_state.h b/ydb/core/keyvalue/keyvalue_state.h index f57a69d35d3..b0e6dcb3a2f 100644 --- a/ydb/core/keyvalue/keyvalue_state.h +++ b/ydb/core/keyvalue/keyvalue_state.h @@ -28,33 +28,33 @@ namespace NKikimr { namespace NKeyValue { class TKeyValueState { - struct TRangeSet { - TMap<ui64, ui64> EndForBegin; - - TRangeSet() { - } - - void Add(ui64 begin, ui64 end) { - Y_VERIFY(begin < end, "begin# %" PRIu64 " end# %" PRIu64, (ui64)begin, (ui64)end); - EndForBegin[begin] = end; - } - - bool Remove(ui64 x) { - auto it = EndForBegin.upper_bound(x); - if (it != EndForBegin.begin()) { - it--; - } else { - return false; - } - if (x >= it->second) { - return false; - } - Y_VERIFY(it->first <= x); - EndForBegin.erase(it); - return true; - } - }; - + struct TRangeSet { + TMap<ui64, ui64> EndForBegin; + + TRangeSet() { + } + + void Add(ui64 begin, ui64 end) { + Y_VERIFY(begin < end, "begin# %" PRIu64 " end# %" PRIu64, (ui64)begin, (ui64)end); + EndForBegin[begin] = end; + } + + bool Remove(ui64 x) { + auto it = EndForBegin.upper_bound(x); + if (it != EndForBegin.begin()) { + it--; + } else { + return false; + } + if (x >= it->second) { + return false; + } + Y_VERIFY(it->first <= x); + EndForBegin.erase(it); + return true; + } + }; + public: using TIndex = TMap<TString, TIndexRecord>; using TCommand = NKikimrKeyValue::ExecuteTransactionRequest::Command; @@ -243,7 +243,7 @@ protected: using TKeySet = TIncrementalKeySet; - TVector<TRangeSet> ChannelRangeSets; + TVector<TRangeSet> ChannelRangeSets; TIndex Index; THashMap<TLogoBlobID, ui32> RefCounts; TSet<TLogoBlobID> Trash; @@ -266,7 +266,7 @@ protected: bool IsTabletYellowMove; bool IsTabletYellowStop; TActorId ChannelBalancerActorId; - ui64 InitialCollectsSent = 0; + ui64 InitialCollectsSent = 0; TDeque<TAutoPtr<TIntermediate>> Queue; ui64 IntermediatesInFlight; @@ -312,26 +312,26 @@ public: void Load(const TString &key, const TString& value); void InitExecute(ui64 tabletId, TActorId keyValueActorId, ui32 executorGeneration, ISimpleDb &db, const TActorContext &ctx, const TTabletStorageInfo *info); - void RegisterInitialCollectResult(const TActorContext &ctx); - void SendCutHistory(const TActorContext &ctx); + void RegisterInitialCollectResult(const TActorContext &ctx); + void SendCutHistory(const TActorContext &ctx); void OnInitQueueEmpty(const TActorContext &ctx); void OnStateWork(const TActorContext &ctx); void RequestExecute(THolder<TIntermediate> &intermediate, ISimpleDb &db, const TActorContext &ctx, const TTabletStorageInfo *info); - void RequestComplete(THolder<TIntermediate> &intermediate, const TActorContext &ctx, const TTabletStorageInfo *info); - - // garbage collection methods + void RequestComplete(THolder<TIntermediate> &intermediate, const TActorContext &ctx, const TTabletStorageInfo *info); + + // garbage collection methods void PrepareCollectIfNeeded(const TActorContext &ctx); void StoreCollectExecute(ISimpleDb &db, const TActorContext &ctx); void StoreCollectComplete(const TActorContext &ctx); void EraseCollectExecute(ISimpleDb &db, const TActorContext &ctx); void EraseCollectComplete(const TActorContext &ctx); - void SendStoreCollect(const TActorContext &ctx, const THelpers::TGenerationStep &genStep, - TVector<TLogoBlobID> &keep, TVector<TLogoBlobID> &doNotKeep); - void StartCollectingIfPossible(const TActorContext &ctx); - ui64 OnEvCollect(const TActorContext &ctx); - void OnEvCollectDone(ui64 perGenerationCounterStepSize, const TActorContext &ctx); - void OnEvEraseCollect(const TActorContext &ctx); + void SendStoreCollect(const TActorContext &ctx, const THelpers::TGenerationStep &genStep, + TVector<TLogoBlobID> &keep, TVector<TLogoBlobID> &doNotKeep); + void StartCollectingIfPossible(const TActorContext &ctx); + ui64 OnEvCollect(const TActorContext &ctx); + void OnEvCollectDone(ui64 perGenerationCounterStepSize, const TActorContext &ctx); + void OnEvEraseCollect(const TActorContext &ctx); void Reply(THolder<TIntermediate> &intermediate, const TActorContext &ctx, const TTabletStorageInfo *info); void ProcessCmd(TIntermediate::TRead &read, diff --git a/ydb/core/keyvalue/keyvalue_state_collect.cpp b/ydb/core/keyvalue/keyvalue_state_collect.cpp index 604c1e20ffc..b19efefed4f 100644 --- a/ydb/core/keyvalue/keyvalue_state_collect.cpp +++ b/ydb/core/keyvalue/keyvalue_state_collect.cpp @@ -1,220 +1,220 @@ -#include "keyvalue_state.h" - -namespace NKikimr { -namespace NKeyValue { - -void TKeyValueState::PrepareCollectIfNeeded(const TActorContext &ctx) { - LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "PrepareCollectIfNeeded KeyValue# " << TabletId << " Marker# KV61"); - - if (CollectOperation.Get()) { - // We already are trying to collect something, just pass this time. - return; - } - - if (Trash.empty()) { - // There is nothing to collect. - return; - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // calculate maximum blob id in trash - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - TLogoBlobID maxId = *Trash.rbegin(); - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // derive new collect step for this operation - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ui64 collectNeededStep = 0; - if (maxId.Generation() == ExecutorGeneration) { - collectNeededStep = maxId.Step(); // collect up to this maximum blob - } else { - // step 0 by default provides collection of all trash blobs up to maxId as the generation is less than current one - Y_VERIFY(maxId.Generation() < ExecutorGeneration); - } - - // Don't let the CollectGeneration step backwards - if (StoredState.GetCollectGeneration() == ExecutorGeneration && collectNeededStep < StoredState.GetCollectStep()) { - collectNeededStep = StoredState.GetCollectStep(); - } - - // create basic collect operation with zero keep/doNotKeep flag vectors; they will be calculated just before sending - CollectOperation.Reset(new TCollectOperation(ExecutorGeneration, collectNeededStep, {} /* keep */, {} /* doNoKeep */)); - if (collectNeededStep == NextLogoBlobStep) { - // advance to the next step if we are going to collect everything up to current one; otherwise we can keep - // current step - Step(); - } - - StartCollectingIfPossible(ctx); -} - -void TKeyValueState::StoreCollectExecute(ISimpleDb &db, const TActorContext &ctx) { - LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "StoreCollectExecute KeyValue# " << TabletId - << " IsDamaged# " << IsDamaged << " Marker# KV62"); - - if (IsDamaged) { - return; - } - Y_VERIFY(CollectOperation.Get()); - - // This operation will be executed no matter what - const ui32 collectGen = CollectOperation->Header.GetCollectGeneration(); - const ui32 collectStep = CollectOperation->Header.GetCollectStep(); - THelpers::DbUpdateCollect(collectGen, - collectStep, - CollectOperation->Keep, - CollectOperation->DoNotKeep, - db, ctx); - - // We don't need to keep the trash list anymore - for (TLogoBlobID &id: CollectOperation->DoNotKeep) { - THelpers::DbEraseTrash(id, db, ctx); - const ui32 num = Trash.erase(id); - Y_VERIFY(num == 1); - CountTrashCollected(id.BlobSize()); - } - - // remove trash entries that were not marked as 'Keep' before, but which are automatically deleted by this barrier - // to prevent them from being added to 'DoNotKeep' list after - for (auto it = Trash.begin(); it != Trash.end(); ) { - const THelpers::TGenerationStep trashGenStep = THelpers::GenerationStep(*it); - ui64 storedCollectGeneration = StoredState.GetCollectGeneration(); - ui64 storedCollectStep = StoredState.GetCollectStep(); - const bool a = trashGenStep > THelpers::TGenerationStep(storedCollectGeneration, storedCollectStep); - const bool b = trashGenStep <= THelpers::TGenerationStep(collectGen, collectStep); - if (a && b) { - CountTrashCollected(it->BlobSize()); - THelpers::DbEraseTrash(*it, db, ctx); - it = Trash.erase(it); - } else { - ++it; - } - } -} - -void TKeyValueState::StoreCollectComplete(const TActorContext &ctx) { - ctx.Send(KeyValueActorId, new TEvKeyValue::TEvCollect()); -} - -void TKeyValueState::EraseCollectExecute(ISimpleDb &db, const TActorContext &ctx) { - LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "EraseCollectExecute KeyValue# " << TabletId - << " IsDamaged# " << IsDamaged << " Marker# KV63"); - if (IsDamaged) { - return; - } - Y_VERIFY(CollectOperation.Get()); - // Erase the collect operation - THelpers::DbEraseCollect(db, ctx); - // Update the state - StoredState.SetCollectGeneration(CollectOperation->Header.GetCollectGeneration()); - StoredState.SetCollectStep(CollectOperation->Header.GetCollectStep()); - THelpers::DbUpdateState(StoredState, db, ctx); -} - -void TKeyValueState::EraseCollectComplete(const TActorContext &ctx) { - Y_UNUSED(ctx); - Y_VERIFY(CollectOperation.Get()); - CollectOperation.Reset(nullptr); - IsCollectEventSent = false; - - // Start new collect operation if needed - PrepareCollectIfNeeded(ctx); -} - -// Prepare the completely new full collect operation with the same gen/step, but with correct keep & doNotKeep lists -void TKeyValueState::SendStoreCollect(const TActorContext &ctx, const THelpers::TGenerationStep &genStep, - TVector<TLogoBlobID> &keep, TVector<TLogoBlobID> &doNotKeep) { - ui32 generation, step; - std::tie(generation, step) = genStep; - CollectOperation.Reset(new TCollectOperation(generation, step, std::move(keep), std::move(doNotKeep))); - ctx.Send(KeyValueActorId, new TEvKeyValue::TEvStoreCollect()); -} - -void TKeyValueState::StartCollectingIfPossible(const TActorContext &ctx) { - LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "StartCollectingIfPossible KeyValue# " << TabletId - << " IsCollectEventSent# " << IsCollectEventSent << " Marker# KV64"); - - // there is nothing to collect yet, or the event was already sent - if (!CollectOperation || IsCollectEventSent) { - return; - } - - // create generation:step barrier tuple for proposed garbage collection command - const auto &header = CollectOperation->Header; - auto collectGenStep = THelpers::TGenerationStep(header.GetCollectGeneration(), header.GetCollectStep()); - - // if we have some in flight writes, check if they do not overlap with the new barrier - if (InFlightForStep) { - // get the first item from the map with the least value; its key contains step of blob being written - auto leastWriteIt = InFlightForStep.begin(); - // construct generation:step pair of the oldest in flight blob - auto inFlightGenStep = THelpers::TGenerationStep(ExecutorGeneration, leastWriteIt->first); - // check if the new barrier would delete blob being written, in this case hold with the collect operation - if (inFlightGenStep <= collectGenStep) { - return; - } - } - - Y_VERIFY(!IsCollectEventSent); - IsCollectEventSent = true; - - // create stored (previously issued) generation:step barrier as a tuple - auto storedGenStep = THelpers::TGenerationStep(StoredState.GetCollectGeneration(), StoredState.GetCollectStep()); - - // ensure that barrier advances in correct direction - Y_VERIFY(collectGenStep >= storedGenStep); - - // create list of blobs that must have Keep flag - TVector<TLogoBlobID> keep; - for (const auto &kv : RefCounts) { - const TLogoBlobID &id = kv.first; - const THelpers::TGenerationStep genStep = THelpers::GenerationStep(id); - if (storedGenStep < genStep && genStep <= collectGenStep) { - keep.push_back(id); - } - } - - // create list of blobs that must have to DoNotKeep flag set; these blobs must have Keep flag written and reside in - // Trash now - TVector<TLogoBlobID> doNotKeep; - doNotKeep.reserve(Trash.size()); - for (const TLogoBlobID &id : Trash) { - if (THelpers::GenerationStep(id) <= storedGenStep) { - doNotKeep.push_back(id); - } - } - doNotKeep.shrink_to_fit(); - - LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "StartCollectingIfPossible KeyValue# " << TabletId - << "Flags Keep.Size# " << keep.size() << " DoNotKeep.Size# " << doNotKeep.size() << " Marker# KV65"); - - SendStoreCollect(ctx, collectGenStep, keep, doNotKeep); -} - -ui64 TKeyValueState::OnEvCollect(const TActorContext &ctx) { - Y_UNUSED(ctx); - LastCollectStartedAt = TAppData::TimeProvider->Now(); - Y_VERIFY(CollectOperation.Get()); - ui64 perGenerationCounterStepSize = TEvBlobStorage::TEvCollectGarbage::PerGenerationCounterStepSize( - &CollectOperation->Keep, &CollectOperation->DoNotKeep); - ui64 nextPerGenerationCounter = ui64(PerGenerationCounter) + perGenerationCounterStepSize; - if (nextPerGenerationCounter > ui64(Max<ui32>())) { - return 0; - } - return perGenerationCounterStepSize; -} - -void TKeyValueState::OnEvCollectDone(ui64 perGenerationCounterStepSize, const TActorContext &ctx) { - Y_UNUSED(ctx); - Y_VERIFY(perGenerationCounterStepSize >= 1); - PerGenerationCounter += perGenerationCounterStepSize; -} - -void TKeyValueState::OnEvEraseCollect(const TActorContext &ctx) { - Y_UNUSED(ctx); - CountLatencyBsCollect(); -} - -} // NKeyValue -} // NKikimr - +#include "keyvalue_state.h" + +namespace NKikimr { +namespace NKeyValue { + +void TKeyValueState::PrepareCollectIfNeeded(const TActorContext &ctx) { + LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "PrepareCollectIfNeeded KeyValue# " << TabletId << " Marker# KV61"); + + if (CollectOperation.Get()) { + // We already are trying to collect something, just pass this time. + return; + } + + if (Trash.empty()) { + // There is nothing to collect. + return; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // calculate maximum blob id in trash + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + TLogoBlobID maxId = *Trash.rbegin(); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // derive new collect step for this operation + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ui64 collectNeededStep = 0; + if (maxId.Generation() == ExecutorGeneration) { + collectNeededStep = maxId.Step(); // collect up to this maximum blob + } else { + // step 0 by default provides collection of all trash blobs up to maxId as the generation is less than current one + Y_VERIFY(maxId.Generation() < ExecutorGeneration); + } + + // Don't let the CollectGeneration step backwards + if (StoredState.GetCollectGeneration() == ExecutorGeneration && collectNeededStep < StoredState.GetCollectStep()) { + collectNeededStep = StoredState.GetCollectStep(); + } + + // create basic collect operation with zero keep/doNotKeep flag vectors; they will be calculated just before sending + CollectOperation.Reset(new TCollectOperation(ExecutorGeneration, collectNeededStep, {} /* keep */, {} /* doNoKeep */)); + if (collectNeededStep == NextLogoBlobStep) { + // advance to the next step if we are going to collect everything up to current one; otherwise we can keep + // current step + Step(); + } + + StartCollectingIfPossible(ctx); +} + +void TKeyValueState::StoreCollectExecute(ISimpleDb &db, const TActorContext &ctx) { + LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "StoreCollectExecute KeyValue# " << TabletId + << " IsDamaged# " << IsDamaged << " Marker# KV62"); + + if (IsDamaged) { + return; + } + Y_VERIFY(CollectOperation.Get()); + + // This operation will be executed no matter what + const ui32 collectGen = CollectOperation->Header.GetCollectGeneration(); + const ui32 collectStep = CollectOperation->Header.GetCollectStep(); + THelpers::DbUpdateCollect(collectGen, + collectStep, + CollectOperation->Keep, + CollectOperation->DoNotKeep, + db, ctx); + + // We don't need to keep the trash list anymore + for (TLogoBlobID &id: CollectOperation->DoNotKeep) { + THelpers::DbEraseTrash(id, db, ctx); + const ui32 num = Trash.erase(id); + Y_VERIFY(num == 1); + CountTrashCollected(id.BlobSize()); + } + + // remove trash entries that were not marked as 'Keep' before, but which are automatically deleted by this barrier + // to prevent them from being added to 'DoNotKeep' list after + for (auto it = Trash.begin(); it != Trash.end(); ) { + const THelpers::TGenerationStep trashGenStep = THelpers::GenerationStep(*it); + ui64 storedCollectGeneration = StoredState.GetCollectGeneration(); + ui64 storedCollectStep = StoredState.GetCollectStep(); + const bool a = trashGenStep > THelpers::TGenerationStep(storedCollectGeneration, storedCollectStep); + const bool b = trashGenStep <= THelpers::TGenerationStep(collectGen, collectStep); + if (a && b) { + CountTrashCollected(it->BlobSize()); + THelpers::DbEraseTrash(*it, db, ctx); + it = Trash.erase(it); + } else { + ++it; + } + } +} + +void TKeyValueState::StoreCollectComplete(const TActorContext &ctx) { + ctx.Send(KeyValueActorId, new TEvKeyValue::TEvCollect()); +} + +void TKeyValueState::EraseCollectExecute(ISimpleDb &db, const TActorContext &ctx) { + LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "EraseCollectExecute KeyValue# " << TabletId + << " IsDamaged# " << IsDamaged << " Marker# KV63"); + if (IsDamaged) { + return; + } + Y_VERIFY(CollectOperation.Get()); + // Erase the collect operation + THelpers::DbEraseCollect(db, ctx); + // Update the state + StoredState.SetCollectGeneration(CollectOperation->Header.GetCollectGeneration()); + StoredState.SetCollectStep(CollectOperation->Header.GetCollectStep()); + THelpers::DbUpdateState(StoredState, db, ctx); +} + +void TKeyValueState::EraseCollectComplete(const TActorContext &ctx) { + Y_UNUSED(ctx); + Y_VERIFY(CollectOperation.Get()); + CollectOperation.Reset(nullptr); + IsCollectEventSent = false; + + // Start new collect operation if needed + PrepareCollectIfNeeded(ctx); +} + +// Prepare the completely new full collect operation with the same gen/step, but with correct keep & doNotKeep lists +void TKeyValueState::SendStoreCollect(const TActorContext &ctx, const THelpers::TGenerationStep &genStep, + TVector<TLogoBlobID> &keep, TVector<TLogoBlobID> &doNotKeep) { + ui32 generation, step; + std::tie(generation, step) = genStep; + CollectOperation.Reset(new TCollectOperation(generation, step, std::move(keep), std::move(doNotKeep))); + ctx.Send(KeyValueActorId, new TEvKeyValue::TEvStoreCollect()); +} + +void TKeyValueState::StartCollectingIfPossible(const TActorContext &ctx) { + LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "StartCollectingIfPossible KeyValue# " << TabletId + << " IsCollectEventSent# " << IsCollectEventSent << " Marker# KV64"); + + // there is nothing to collect yet, or the event was already sent + if (!CollectOperation || IsCollectEventSent) { + return; + } + + // create generation:step barrier tuple for proposed garbage collection command + const auto &header = CollectOperation->Header; + auto collectGenStep = THelpers::TGenerationStep(header.GetCollectGeneration(), header.GetCollectStep()); + + // if we have some in flight writes, check if they do not overlap with the new barrier + if (InFlightForStep) { + // get the first item from the map with the least value; its key contains step of blob being written + auto leastWriteIt = InFlightForStep.begin(); + // construct generation:step pair of the oldest in flight blob + auto inFlightGenStep = THelpers::TGenerationStep(ExecutorGeneration, leastWriteIt->first); + // check if the new barrier would delete blob being written, in this case hold with the collect operation + if (inFlightGenStep <= collectGenStep) { + return; + } + } + + Y_VERIFY(!IsCollectEventSent); + IsCollectEventSent = true; + + // create stored (previously issued) generation:step barrier as a tuple + auto storedGenStep = THelpers::TGenerationStep(StoredState.GetCollectGeneration(), StoredState.GetCollectStep()); + + // ensure that barrier advances in correct direction + Y_VERIFY(collectGenStep >= storedGenStep); + + // create list of blobs that must have Keep flag + TVector<TLogoBlobID> keep; + for (const auto &kv : RefCounts) { + const TLogoBlobID &id = kv.first; + const THelpers::TGenerationStep genStep = THelpers::GenerationStep(id); + if (storedGenStep < genStep && genStep <= collectGenStep) { + keep.push_back(id); + } + } + + // create list of blobs that must have to DoNotKeep flag set; these blobs must have Keep flag written and reside in + // Trash now + TVector<TLogoBlobID> doNotKeep; + doNotKeep.reserve(Trash.size()); + for (const TLogoBlobID &id : Trash) { + if (THelpers::GenerationStep(id) <= storedGenStep) { + doNotKeep.push_back(id); + } + } + doNotKeep.shrink_to_fit(); + + LOG_TRACE_S(ctx, NKikimrServices::KEYVALUE, "StartCollectingIfPossible KeyValue# " << TabletId + << "Flags Keep.Size# " << keep.size() << " DoNotKeep.Size# " << doNotKeep.size() << " Marker# KV65"); + + SendStoreCollect(ctx, collectGenStep, keep, doNotKeep); +} + +ui64 TKeyValueState::OnEvCollect(const TActorContext &ctx) { + Y_UNUSED(ctx); + LastCollectStartedAt = TAppData::TimeProvider->Now(); + Y_VERIFY(CollectOperation.Get()); + ui64 perGenerationCounterStepSize = TEvBlobStorage::TEvCollectGarbage::PerGenerationCounterStepSize( + &CollectOperation->Keep, &CollectOperation->DoNotKeep); + ui64 nextPerGenerationCounter = ui64(PerGenerationCounter) + perGenerationCounterStepSize; + if (nextPerGenerationCounter > ui64(Max<ui32>())) { + return 0; + } + return perGenerationCounterStepSize; +} + +void TKeyValueState::OnEvCollectDone(ui64 perGenerationCounterStepSize, const TActorContext &ctx) { + Y_UNUSED(ctx); + Y_VERIFY(perGenerationCounterStepSize >= 1); + PerGenerationCounter += perGenerationCounterStepSize; +} + +void TKeyValueState::OnEvEraseCollect(const TActorContext &ctx) { + Y_UNUSED(ctx); + CountLatencyBsCollect(); +} + +} // NKeyValue +} // NKikimr + diff --git a/ydb/core/keyvalue/keyvalue_storage_request.cpp b/ydb/core/keyvalue/keyvalue_storage_request.cpp index e9032d2ed3d..536b7c281e1 100644 --- a/ydb/core/keyvalue/keyvalue_storage_request.cpp +++ b/ydb/core/keyvalue/keyvalue_storage_request.cpp @@ -35,9 +35,9 @@ class TKeyValueStorageRequest : public TActorBootstrapped<TKeyValueStorageReques THPTimer PutTimer; TInstant GetStatusSentAt; - TInstant LastSuccess; - const TDuration MuteDuration = TDuration::Seconds(5); - TLogPriorityMuteChecker<NLog::PRI_DEBUG, NLog::PRI_ERROR> ErrorStateMuteChecker; + TInstant LastSuccess; + const TDuration MuteDuration = TDuration::Seconds(5); + TLogPriorityMuteChecker<NLog::PRI_DEBUG, NLog::PRI_ERROR> ErrorStateMuteChecker; THashMap<ui32, TInstant> TimeForNextSend; @@ -116,26 +116,26 @@ public: NKikimrProto::EReplyStatus status = ev->Get()->Status; if (status != NKikimrProto::OK) { - TInstant now = TAppData::TimeProvider->Now(); - + TInstant now = TAppData::TimeProvider->Now(); + TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " Unexpected EvPut result# " << NKikimrProto::EReplyStatus_Name(status).data(); str << " Deadline# " << IntermediateResults->Deadline.MilliSeconds(); - str << " Now# " << now.MilliSeconds(); + str << " Now# " << now.MilliSeconds(); str << " duration# " << duration; str << " GotAt# " << IntermediateResults->Stat.IntermediateCreatedAt.MilliSeconds(); - str << " LastSuccess# " << LastSuccess; - str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); + str << " LastSuccess# " << LastSuccess; + str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); str << " EnqueuedAs# " << IntermediateResults->Stat.EnqueuedAs; - str << " ErrorReason# " << ev->Get()->ErrorReason; + str << " ErrorReason# " << ev->Get()->ErrorReason; str << " Marker# KV24"; - - NLog::EPriority logPriority = ErrorStateMuteChecker.Register(now, MuteDuration); - + + NLog::EPriority logPriority = ErrorStateMuteChecker.Register(now, MuteDuration); + ReplyErrorAndDie(ctx, str.Str(), - status == NKikimrProto::TIMEOUT ? NMsgBusProxy::MSTATUS_TIMEOUT : NMsgBusProxy::MSTATUS_INTERNALERROR, - logPriority); + status == NKikimrProto::TIMEOUT ? NMsgBusProxy::MSTATUS_TIMEOUT : NMsgBusProxy::MSTATUS_INTERNALERROR, + logPriority); return; } ui64 cookie = ev->Cookie; @@ -165,8 +165,8 @@ public: } void Handle(TEvBlobStorage::TEvStatusResult::TPtr &ev, const TActorContext &ctx) { - TInstant now = TAppData::TimeProvider->Now(); - ui64 durationMs = (now - GetStatusSentAt).MilliSeconds(); + TInstant now = TAppData::TimeProvider->Now(); + ui64 durationMs = (now - GetStatusSentAt).MilliSeconds(); IntermediateResults->Stat.GetStatusLatencies.push_back(durationMs); ui64 cookie = ev->Cookie; // groupId @@ -178,13 +178,13 @@ public: str << "KeyValue# " << TabletInfo->TabletID; str << " Unexpected EvStatusResult# " << NKikimrProto::EReplyStatus_Name(status).data(); str << " Deadline# " << IntermediateResults->Deadline.MilliSeconds(); - str << " Now# " << now.MilliSeconds(); + str << " Now# " << now.MilliSeconds(); str << " GetStatusSentAt# " << GetStatusSentAt.MilliSeconds(); str << " GotAt# " << IntermediateResults->Stat.IntermediateCreatedAt.MilliSeconds(); - str << " LastSuccess# " << LastSuccess; - str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); + str << " LastSuccess# " << LastSuccess; + str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); str << " EnqueuedAs# " << IntermediateResults->Stat.EnqueuedAs; - str << " ErrorReason# " << ev->Get()->ErrorReason; + str << " ErrorReason# " << ev->Get()->ErrorReason; str << " Marker# KV28"; ReplyErrorAndDie(ctx, str.Str(), status == NKikimrProto::TIMEOUT ? NMsgBusProxy::MSTATUS_TIMEOUT : NMsgBusProxy::MSTATUS_INTERNALERROR); @@ -195,7 +195,7 @@ public: TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " EvStatus cookie# " << (ui64)cookie; - str << " > GetStatuses# " << (ui32)IntermediateResults->GetStatuses.size(); + str << " > GetStatuses# " << (ui32)IntermediateResults->GetStatuses.size(); str << " Marker# KV29"; ReplyErrorAndDie(ctx, str.Str()); return; @@ -241,17 +241,17 @@ public: NKikimrProto::EReplyStatus status = ev->Get()->Status; if (status != NKikimrProto::OK) { - TInstant now = TAppData::TimeProvider->Now(); - + TInstant now = TAppData::TimeProvider->Now(); + TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " Unexpected EvGet result# " << NKikimrProto::EReplyStatus_Name(status).data(); str << " Deadline# " << IntermediateResults->Deadline.MilliSeconds(); - str << " Now# " << now.MilliSeconds(); + str << " Now# " << now.MilliSeconds(); str << " SentAt# " << request.SentAt.MilliSeconds(); str << " GotAt# " << IntermediateResults->Stat.IntermediateCreatedAt.MilliSeconds(); - str << " LastSuccess# " << LastSuccess; - str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); + str << " LastSuccess# " << LastSuccess; + str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); str << " EnqueuedAs# " << IntermediateResults->Stat.EnqueuedAs; str << " ErrorReason# " << ev->Get()->ErrorReason; str << " Marker# KV26"; @@ -318,19 +318,19 @@ public: } bool UpdateRequest(const TActorContext &ctx) { - TInstant now = TAppData::TimeProvider->Now(); - if (IntermediateResults->Deadline <= now) { + TInstant now = TAppData::TimeProvider->Now(); + if (IntermediateResults->Deadline <= now) { TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " Deadline reached Deadline# " << IntermediateResults->Deadline.MilliSeconds(); - str << " Now# " << now.MilliSeconds(); + str << " Now# " << now.MilliSeconds(); str << " GotAt# " << IntermediateResults->Stat.IntermediateCreatedAt.MilliSeconds(); str << " EnqueuedAs# " << IntermediateResults->Stat.EnqueuedAs; str << " Marker# KV30"; ReplyErrorAndDie(ctx, str.Str(), NMsgBusProxy::MSTATUS_TIMEOUT); return true; } - LastSuccess = now; + LastSuccess = now; bool sentSomething = false; while (SendSomeReadRequests(ctx)) { sentSomething = true; @@ -403,16 +403,16 @@ public: void Handle(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { Y_UNUSED(ev); - TInstant now = TAppData::TimeProvider->Now(); - + TInstant now = TAppData::TimeProvider->Now(); + TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " Deadline reached before answering the request."; str << " Deadline# " << IntermediateResults->Deadline.MilliSeconds(); - str << " Now# " << now.MilliSeconds(); + str << " Now# " << now.MilliSeconds(); str << " GotAt# " << IntermediateResults->Stat.IntermediateCreatedAt.MilliSeconds(); - str << " LastSuccess# " << LastSuccess; - str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); + str << " LastSuccess# " << LastSuccess; + str << " SinceLastSuccess# " << (now - LastSuccess).MilliSeconds(); str << " EnqueuedAs# " << IntermediateResults->Stat.EnqueuedAs; str << " Marker# KV312"; ReplyErrorAndDie(ctx, str.Str(), NMsgBusProxy::MSTATUS_TIMEOUT); @@ -422,13 +422,13 @@ public: void Bootstrap(const TActorContext &ctx) { // Check parameters and send requests if (IntermediateResults->Deadline != TInstant::Max()) { - TInstant now = TAppData::TimeProvider->Now(); - if (IntermediateResults->Deadline <= now) { + TInstant now = TAppData::TimeProvider->Now(); + if (IntermediateResults->Deadline <= now) { TStringStream str; str << "KeyValue# " << TabletInfo->TabletID; str << " Deadline reached before processing request."; str << " Deadline# " << IntermediateResults->Deadline.MilliSeconds(); - str << " Now# " << now.MilliSeconds(); + str << " Now# " << now.MilliSeconds(); str << " GotAt# " << IntermediateResults->Stat.IntermediateCreatedAt.MilliSeconds(); str << " EnqueuedAs# " << IntermediateResults->Stat.EnqueuedAs; str << " Marker# KV311"; @@ -436,7 +436,7 @@ public: return; } - const TDuration timeout = IntermediateResults->Deadline - now; + const TDuration timeout = IntermediateResults->Deadline - now; ctx.Schedule(timeout, new TEvents::TEvWakeup()); } @@ -458,10 +458,10 @@ public: } void ReplyErrorAndDie(const TActorContext &ctx, TString errorDescription, - NMsgBusProxy::EResponseStatus status = NMsgBusProxy::MSTATUS_INTERNALERROR, - NLog::EPriority logPriority = NLog::PRI_ERROR) { - LOG_LOG_S(ctx, logPriority, NKikimrServices::KEYVALUE, errorDescription); - + NMsgBusProxy::EResponseStatus status = NMsgBusProxy::MSTATUS_INTERNALERROR, + NLog::EPriority logPriority = NLog::PRI_ERROR) { + LOG_LOG_S(ctx, logPriority, NKikimrServices::KEYVALUE, errorDescription); + THolder<TEvKeyValue::TEvResponse> response(new TEvKeyValue::TEvResponse); if (IntermediateResults->HasCookie) { response->Record.SetCookie(IntermediateResults->Cookie); diff --git a/ydb/core/keyvalue/keyvalue_ut.cpp b/ydb/core/keyvalue/keyvalue_ut.cpp index 4ae1fcf56a6..ff88e52b1e1 100644 --- a/ydb/core/keyvalue/keyvalue_ut.cpp +++ b/ydb/core/keyvalue/keyvalue_ut.cpp @@ -884,30 +884,30 @@ Y_UNIT_TEST(TestWriteReadDeleteWithRestartsThenResponseOkWithNewApi) { } -Y_UNIT_TEST(TestRewriteThenLastValue) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - TVector<NKikimrClient::TKeyValueRequest::EStorageChannel> channels = - {NKikimrClient::TKeyValueRequest::MAIN, NKikimrClient::TKeyValueRequest::INLINE}; - for (auto &ch1 : channels) { - for (auto &ch2 : channels) { - CmdWrite("key", "value", ch1, - NKikimrClient::TKeyValueRequest::REALTIME, tc); - CmdRead({"key"}, NKikimrClient::TKeyValueRequest::REALTIME, - {"value"}, {}, tc); - CmdWrite("key", "updated", ch2, - NKikimrClient::TKeyValueRequest::REALTIME, tc); - CmdRead({"key"}, NKikimrClient::TKeyValueRequest::REALTIME, - {"updated"}, {}, tc); - } - } - }); -} - +Y_UNIT_TEST(TestRewriteThenLastValue) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + TVector<NKikimrClient::TKeyValueRequest::EStorageChannel> channels = + {NKikimrClient::TKeyValueRequest::MAIN, NKikimrClient::TKeyValueRequest::INLINE}; + for (auto &ch1 : channels) { + for (auto &ch2 : channels) { + CmdWrite("key", "value", ch1, + NKikimrClient::TKeyValueRequest::REALTIME, tc); + CmdRead({"key"}, NKikimrClient::TKeyValueRequest::REALTIME, + {"value"}, {}, tc); + CmdWrite("key", "updated", ch2, + NKikimrClient::TKeyValueRequest::REALTIME, tc); + CmdRead({"key"}, NKikimrClient::TKeyValueRequest::REALTIME, + {"updated"}, {}, tc); + } + } + }); +} + Y_UNIT_TEST(TestRewriteThenLastValueNewApi) { TTestContext tc; @@ -2064,69 +2064,69 @@ Y_UNIT_TEST(TestSetExecutorFastLogPolicy) { }); } -Y_UNIT_TEST(TestWriteDeleteThenReadRemaining) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); +Y_UNIT_TEST(TestWriteDeleteThenReadRemaining) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString &dispatchName, std::function<void(TTestActorRuntime&)> setup, bool &activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); activeZone = false; - const ui32 BatchCount = 5; - const ui32 ItemsPerBatch = 50; - TDeque<TString> allKeys; - TDeque<TString> allValues; - for (ui32 i = 0; i < BatchCount; ++i) { - TDeque<TString> batchKeys; - TDeque<TString> batchValues; - for (ui32 j = 0; j < ItemsPerBatch; ++j) { - TString key = Sprintf("k%08" PRIu32, i * ItemsPerBatch + j); - TString val = Sprintf("v%08" PRIu32, i * ItemsPerBatch + j); - batchKeys.push_back(key); - batchValues.push_back(val); - allKeys.push_back(key); - allValues.push_back(val); - } - CmdWrite(batchKeys, batchValues, - NKikimrClient::TKeyValueRequest::INLINE, - NKikimrClient::TKeyValueRequest::REALTIME, tc); - } - { + const ui32 BatchCount = 5; + const ui32 ItemsPerBatch = 50; + TDeque<TString> allKeys; + TDeque<TString> allValues; + for (ui32 i = 0; i < BatchCount; ++i) { + TDeque<TString> batchKeys; + TDeque<TString> batchValues; + for (ui32 j = 0; j < ItemsPerBatch; ++j) { + TString key = Sprintf("k%08" PRIu32, i * ItemsPerBatch + j); + TString val = Sprintf("v%08" PRIu32, i * ItemsPerBatch + j); + batchKeys.push_back(key); + batchValues.push_back(val); + allKeys.push_back(key); + allValues.push_back(val); + } + CmdWrite(batchKeys, batchValues, + NKikimrClient::TKeyValueRequest::INLINE, + NKikimrClient::TKeyValueRequest::REALTIME, tc); + } + { TDesiredPair<TEvKeyValue::TEvRequest> dp; - AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, - allKeys, allValues, NKikimrProto::OK, tc, dp); - RunRequest(dp, tc, __LINE__); - } - - ui32 deleteBound = 50; // just for speedup - - TReallyFastRng32 rng(1); - TDeque<TString> aliveKeys; - TDeque<TString> aliveValues; - while (!allKeys.empty()) { - TString key = allKeys.front(); - TString val = allValues.front(); - allKeys.pop_front(); - allValues.pop_front(); - if (deleteBound && rng() % 2) { - CmdDeleteRange(key, true, key, true, tc); // one key per delete - CmdRead({key}, NKikimrClient::TKeyValueRequest::REALTIME, {""}, {true}, tc); - deleteBound--; - } else { - aliveKeys.push_back(key); - aliveValues.push_back(val); - } - } - // Cerr << "total keys: " << BatchCount * ItemsPerBatch << ", after deletes: " << aliveKeys.size() << Endl; + AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, + allKeys, allValues, NKikimrProto::OK, tc, dp); + RunRequest(dp, tc, __LINE__); + } + + ui32 deleteBound = 50; // just for speedup + + TReallyFastRng32 rng(1); + TDeque<TString> aliveKeys; + TDeque<TString> aliveValues; + while (!allKeys.empty()) { + TString key = allKeys.front(); + TString val = allValues.front(); + allKeys.pop_front(); + allValues.pop_front(); + if (deleteBound && rng() % 2) { + CmdDeleteRange(key, true, key, true, tc); // one key per delete + CmdRead({key}, NKikimrClient::TKeyValueRequest::REALTIME, {""}, {true}, tc); + deleteBound--; + } else { + aliveKeys.push_back(key); + aliveValues.push_back(val); + } + } + // Cerr << "total keys: " << BatchCount * ItemsPerBatch << ", after deletes: " << aliveKeys.size() << Endl; activeZone = true; - { + { TDesiredPair<TEvKeyValue::TEvRequest> dp; - AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, - aliveKeys, aliveValues, NKikimrProto::OK, tc, dp); - RunRequest(dp, tc, __LINE__); - } - }); -} + AddCmdReadRange("a", true, "z", false, true, Max<ui64>(), NKikimrClient::TKeyValueRequest::REALTIME, + aliveKeys, aliveValues, NKikimrProto::OK, tc, dp); + RunRequest(dp, tc, __LINE__); + } + }); +} Y_UNIT_TEST(TestObtainLockNewApi) { diff --git a/ydb/core/keyvalue/ya.make b/ydb/core/keyvalue/ya.make index 17bb1853c05..a083554b113 100644 --- a/ydb/core/keyvalue/ya.make +++ b/ydb/core/keyvalue/ya.make @@ -37,7 +37,7 @@ SRCS( keyvalue_simple_db.h keyvalue_state.cpp keyvalue_state.h - keyvalue_state_collect.cpp + keyvalue_state_collect.cpp keyvalue_storage_read_request.cpp keyvalue_storage_request.cpp keyvalue_storage_request.h diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp index f49aff031f1..19d12125621 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp @@ -64,7 +64,7 @@ TExprBase KqpBuildReadTableStage(TExprBase node, TExprContext& ctx, const TKqpOp TVector<TExprBase> values; TNodeOnNodeOwnedMap replaceMap; - auto checkRange = [&values](const TVarArgCallable<TExprBase>& tuple) { + auto checkRange = [&values](const TVarArgCallable<TExprBase>& tuple) { for (const auto& value : tuple) { if (!IsDqPureExpr(value)) { return false; diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp index fa0bb9cc808..6f7fea17c50 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp @@ -230,11 +230,11 @@ bool IsComparableTypes(const TExprBase& leftNode, const TExprBase& rightNode, bo TVector<std::pair<TExprBase, TExprBase>> ExtractComparisonParameters(const TCoCompare& predicate, - const TExprNode* rawLambdaArg, const TExprBase& input) + const TExprNode* rawLambdaArg, const TExprBase& input) { TVector<std::pair<TExprBase, TExprBase>> out; - auto convertNode = [rawLambdaArg](const TExprBase& node) -> TMaybeNode<TExprBase> { + auto convertNode = [rawLambdaArg](const TExprBase& node) -> TMaybeNode<TExprBase> { if (node.Maybe<TCoNull>()) { return node; } @@ -539,7 +539,7 @@ TMaybeNode<TExprBase> CoalescePushdown(const TCoCoalesce& coalesce, TExprContext return NullNode; } - auto parameters = ExtractComparisonParameters(predicate, lambdaArg, input); + auto parameters = ExtractComparisonParameters(predicate, lambdaArg, input); if (parameters.empty()) { return NullNode; diff --git a/ydb/core/mind/bscontroller/migrate.cpp b/ydb/core/mind/bscontroller/migrate.cpp index 20d10f8b621..90fd322fc52 100644 --- a/ydb/core/mind/bscontroller/migrate.cpp +++ b/ydb/core/mind/bscontroller/migrate.cpp @@ -32,13 +32,13 @@ class TBlobStorageController::TTxMigrate : public TTransactionBase<TBlobStorageC bool Execute(TTransactionContext& txc, const TActorContext&) override { auto& front = Queue.front(); - STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXM03, "Execute tx from queue", (Type, TypeName(*front))); + STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXM03, "Execute tx from queue", (Type, TypeName(*front))); return front->Execute(txc); } void Complete(const TActorContext&) override { auto& front = Queue.front(); - STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXM04, "Complete tx from queue", (Type, TypeName(*front))); + STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXM04, "Complete tx from queue", (Type, TypeName(*front))); front->Complete(); Queue.pop_front(); if (Queue) { diff --git a/ydb/core/mind/bscontroller/monitoring.cpp b/ydb/core/mind/bscontroller/monitoring.cpp index 245c782cea6..2b3758bbf76 100644 --- a/ydb/core/mind/bscontroller/monitoring.cpp +++ b/ydb/core/mind/bscontroller/monitoring.cpp @@ -391,17 +391,17 @@ public: TString GenerateHtml() { TStringStream s; - Self->RenderHeader(s); - + Self->RenderHeader(s); + HTML(s) { ui64 offset = Offset; H3() { - s << "Health events"; - } - H3() { - s << "Health-related operations since " << Since.ToRfc822StringLocal(); + s << "Health events"; } + H3() { + s << "Health-related operations since " << Since.ToRfc822StringLocal(); + } TABLE_CLASS("table") { TABLEHEAD() { TABLER() { @@ -418,7 +418,7 @@ public: const TEvent& event = *it; TABLER() { TABLED() { s << event.Timestamp.ToStringUpToSeconds(); } - TABLED() { s << event.Timestamp.ToRfc822StringLocal(); } + TABLED() { s << event.Timestamp.ToRfc822StringLocal(); } TABLED() { s << event.Message; } TABLED() { for (const TReassignItem& item : event.Reassign) { @@ -457,8 +457,8 @@ public: renderPageLink("Last page", lastPageOffset); } } - - Self->RenderFooter(s); + + Self->RenderFooter(s); return s.Str(); } @@ -643,8 +643,8 @@ bool TBlobStorageController::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr e Execute(new TTxMonEvent_HealthEvents(ev->Sender, cgi, this)); return true; } else if (page == "SelfHeal") { - bool hiddenAction = cgi.Has("action") && cgi.Get("action") == "disableSelfHeal"; - if (cgi.Has("disable") && cgi.Get("disable") == "1" && hiddenAction) { + bool hiddenAction = cgi.Has("action") && cgi.Get("action") == "disableSelfHeal"; + if (cgi.Has("disable") && cgi.Get("disable") == "1" && hiddenAction) { Register(new TDisableSelfHealActor(ev->Sender, TStringBuilder() << "?TabletID=" << TabletID() << "&page=SelfHeal")); } else { diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index b732d98e937..1ca0968786b 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -377,9 +377,9 @@ namespace NKikimr::NBsController { const TInstant now = TActivationContext::Now(); HTML(out) { - H2() { - out << "BlobStorage Controller"; - } + H2() { + out << "BlobStorage Controller"; + } DIV_CLASS("panel panel-info") { DIV_CLASS("panel-heading") { out << "Self-Heal status"; @@ -387,14 +387,14 @@ namespace NKikimr::NBsController { DIV_CLASS("panel-body") { out << (selfHealEnabled ? "Enabled" : "Disabled"); if (selfHealEnabled) { - out << "<br/>" << Endl; - out << "<form method='POST'>" << Endl; - out << "<input type='hidden' name='TabletID' value='" << TabletId << "'>" << Endl; - out << "<input type='hidden' name='page' value='SelfHeal'>" << Endl; - out << "<input type='hidden' name='disable' value='1'>" << Endl; - out << "<input type='hidden' name='action' value='disableSelfHeal'>" << Endl; - out << "<input class='btn btn-primary' type='submit' value='DISABLE NOW'/>" << Endl; - out << "</form>"; + out << "<br/>" << Endl; + out << "<form method='POST'>" << Endl; + out << "<input type='hidden' name='TabletID' value='" << TabletId << "'>" << Endl; + out << "<input type='hidden' name='page' value='SelfHeal'>" << Endl; + out << "<input type='hidden' name='disable' value='1'>" << Endl; + out << "<input type='hidden' name='action' value='disableSelfHeal'>" << Endl; + out << "<input class='btn btn-primary' type='submit' value='DISABLE NOW'/>" << Endl; + out << "</form>"; } } } diff --git a/ydb/core/tablet/tablet_req_blockbs.cpp b/ydb/core/tablet/tablet_req_blockbs.cpp index 7e2e94ec95d..18d5a0694c6 100644 --- a/ydb/core/tablet/tablet_req_blockbs.cpp +++ b/ydb/core/tablet/tablet_req_blockbs.cpp @@ -21,9 +21,9 @@ class TTabletReqBlockBlobStorageGroup : public TActorBootstrapped<TTabletReqBloc void SendRequest() { const TActorId proxy = MakeBlobStorageProxyID(GroupId); - THolder<TEvBlobStorage::TEvBlock> event(new TEvBlobStorage::TEvBlock(TabletId, Generation, TInstant::Max())); - event->IsMonitored = false; - SendToBSProxy(TlsActivationContext->AsActorContext(), proxy, event.Release()); + THolder<TEvBlobStorage::TEvBlock> event(new TEvBlobStorage::TEvBlock(TabletId, Generation, TInstant::Max())); + event->IsMonitored = false; + SendToBSProxy(TlsActivationContext->AsActorContext(), proxy, event.Release()); } void Handle(TEvents::TEvUndelivered::TPtr&) { diff --git a/ydb/core/tablet/tablet_req_delete.cpp b/ydb/core/tablet/tablet_req_delete.cpp index 754c79bb325..043ffe08ff7 100644 --- a/ydb/core/tablet/tablet_req_delete.cpp +++ b/ydb/core/tablet/tablet_req_delete.cpp @@ -65,7 +65,7 @@ class TTabletReqDelete : public TActorBootstrapped<TTabletReqDelete> { Generation, // collectGeneration std::numeric_limits<ui32>::max(), // collectStep TInstant::Max()); // deadline - event->IsMonitored = false; + event->IsMonitored = false; SendToBSProxy(ctx, info.GroupId, event.Release(), numRequest); } diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 27a260bcb27..886ecc69b24 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -4,7 +4,7 @@ #include <ydb/core/util/tuples.h> #include <ydb/core/util/templates.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/unaligned_mem.h> #include <library/cpp/containers/stack_vector/stack_vec.h> #include <utility> diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 59afc462158..d6fff41eebb 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -1404,7 +1404,7 @@ void TExecutor::Execute(TAutoPtr<ITransaction> self, const TActorContext &ctx) { TAutoPtr<TSeat> seat = new TSeat(++TransactionUniqCounter, self); - LWTRACK(TransactionBegin, seat->Self->Orbit, seat->UniqID, Owner->TabletID(), TypeName(*seat->Self)); + LWTRACK(TransactionBegin, seat->Self->Orbit, seat->UniqID, Owner->TabletID(), TypeName(*seat->Self)); ++Stats->TxInFly; Counters->Simple()[TExecutorCounters::DB_TX_IN_FLY] = Stats->TxInFly; diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h index 075ebce354b..978294c84a7 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.h +++ b/ydb/core/tablet_flat/tablet_flat_executor.h @@ -7,7 +7,7 @@ #include <ydb/core/base/blobstorage.h> #include <library/cpp/lwtrace/shuttle.h> #include <util/generic/maybe.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/generic/variant.h> //////////////////////////////////////////// @@ -277,7 +277,7 @@ public: virtual void Describe(IOutputStream &out) const noexcept { - out << TypeName(*this); + out << TypeName(*this); } public: diff --git a/ydb/core/tablet_flat/test/libs/exec/dummy.h b/ydb/core/tablet_flat/test/libs/exec/dummy.h index 5c6e2f5b4f1..e96b25a53cc 100644 --- a/ydb/core/tablet_flat/test/libs/exec/dummy.h +++ b/ydb/core/tablet_flat/test/libs/exec/dummy.h @@ -4,7 +4,7 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <library/cpp/actors/core/actor.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NKikimr { namespace NFake { @@ -80,13 +80,13 @@ namespace NFake { } else if (eh->CastAsLocal<TEvTabletPipe::TEvServerDisconnected>()){ } else if (!TTabletExecutedFlat::HandleDefaultEvents(eh, ctx)) { - Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); + Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); } } void Enqueue(TEventHandlePtr &eh, const ::NActors::TActorContext&) override { - const auto *name = TypeName(*eh->GetBase()).c_str(); + const auto *name = TypeName(*eh->GetBase()).c_str(); Y_FAIL("Got unexpected event %s on tablet booting", name); } diff --git a/ydb/core/tablet_flat/test/libs/exec/leader.h b/ydb/core/tablet_flat/test/libs/exec/leader.h index f964c4b33c3..9e12c2ff97c 100644 --- a/ydb/core/tablet_flat/test/libs/exec/leader.h +++ b/ydb/core/tablet_flat/test/libs/exec/leader.h @@ -70,7 +70,7 @@ namespace NFake { } else if (eh->CastAsLocal<NFake::TEvTerm>()) { } else { - Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); + Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); } } @@ -112,7 +112,7 @@ namespace NFake { } else if (auto logl = Logger->Log(ELnLev::Abort)) { logl - << "Cannot register " << TypeName(*cmd.Actor) << " on " + << "Cannot register " << TypeName(*cmd.Actor) << " on " << " level " << level << ", head " << Head << ".." << Edge; } } diff --git a/ydb/core/tablet_flat/test/libs/exec/nanny.h b/ydb/core/tablet_flat/test/libs/exec/nanny.h index 9b00928c8e4..46f693f8a96 100644 --- a/ydb/core/tablet_flat/test/libs/exec/nanny.h +++ b/ydb/core/tablet_flat/test/libs/exec/nanny.h @@ -134,7 +134,7 @@ namespace NFake { } else if (eh->CastAsLocal<TEvents::TEvPoison>()) { DoSuicide(); } else { - Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); + Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); } } diff --git a/ydb/core/tablet_flat/test/libs/exec/owner.h b/ydb/core/tablet_flat/test/libs/exec/owner.h index f3e3d7363c8..255c69cf7d4 100644 --- a/ydb/core/tablet_flat/test/libs/exec/owner.h +++ b/ydb/core/tablet_flat/test/libs/exec/owner.h @@ -2,7 +2,7 @@ #include "world.h" #include "events.h" -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <library/cpp/actors/core/actor.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/blobstorage.h> @@ -77,7 +77,7 @@ namespace NFake { Send(Agent, new TEvents::TEvPoison); } else { - Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); + Y_Fail("Unexpected event " << TypeName(*eh->GetBase())); } } diff --git a/ydb/core/tablet_flat/test/libs/exec/warden.h b/ydb/core/tablet_flat/test/libs/exec/warden.h index cc2afbde883..a36221e9f0a 100644 --- a/ydb/core/tablet_flat/test/libs/exec/warden.h +++ b/ydb/core/tablet_flat/test/libs/exec/warden.h @@ -6,7 +6,7 @@ #include <ydb/core/base/blobstorage.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/tablet_flat/util_fmt_logger.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <array> @@ -54,7 +54,7 @@ namespace NFake { if (auto logl = Logger->Log(ELnLev::Abort)) { logl << "BS group " << group << " is not configured" - << ", ev " << TypeName(*eh->GetBase()); + << ", ev " << TypeName(*eh->GetBase()); } return; /* cannot process unknown groups */ @@ -70,7 +70,7 @@ namespace NFake { if (auto logl = Logger->Log(ELnLev::Crit)) { logl << "BS group " << group << " is unavailable" - << ", ev " << TypeName(*eh->GetBase()); + << ", ev " << TypeName(*eh->GetBase()); } auto why = TEvents::TEvUndelivered::ReasonActorUnknown; diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index a853040b932..cc16fa3afcd 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -301,7 +301,7 @@ namespace NKikimr { return TTestActorRuntime::EEventAction::PROCESS; HasReboot0 = true; - TString eventType = (event->HasEvent() && event->GetBase()) ? TypeName(*event->GetBase()) : "nullptr"; + TString eventType = (event->HasEvent() && event->GetBase()) ? TypeName(*event->GetBase()) : "nullptr"; if (KillOnCommit && IsCommitResult(event)) { if (ENABLE_REBOOT_DISPATCH_LOG) diff --git a/ydb/core/tx/schemeshard/operation_queue_timer.h b/ydb/core/tx/schemeshard/operation_queue_timer.h index 655fde7c18e..0f90f9f4ccb 100644 --- a/ydb/core/tx/schemeshard/operation_queue_timer.h +++ b/ydb/core/tx/schemeshard/operation_queue_timer.h @@ -93,8 +93,8 @@ struct TShardCompactionInfo { , SearchHeight(searchHeight) {} - TShardCompactionInfo(const TShardCompactionInfo&) = default; - + TShardCompactionInfo(const TShardCompactionInfo&) = default; + bool operator ==(const TShardCompactionInfo& rhs) const { // note that only identity intentionally checked return ShardIdx == rhs.ShardIdx; diff --git a/ydb/core/tx/schemeshard/schemeshard_ui64id.h b/ydb/core/tx/schemeshard/schemeshard_ui64id.h index 26965918c8e..c1834c9f03f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_ui64id.h +++ b/ydb/core/tx/schemeshard/schemeshard_ui64id.h @@ -4,7 +4,7 @@ #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <util/system/types.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/stream/output.h> namespace NKikimr { diff --git a/ydb/core/util/interval_set.h b/ydb/core/util/interval_set.h index 68f6ea66589..2b2d761bf4e 100644 --- a/ydb/core/util/interval_set.h +++ b/ydb/core/util/interval_set.h @@ -64,7 +64,7 @@ struct TIntervalMap { } bool IsSubsetOf(const TIntervalSet<T>& b) const { - return std::visit([this](auto& var) -> bool { return this->IsSubsetOf(var); }, b.Var); + return std::visit([this](auto& var) -> bool { return this->IsSubsetOf(var); }, b.Var); } template <class B> @@ -403,7 +403,7 @@ struct TIntervalVec { } bool IsSubsetOf(const TIntervalSet<T>& b) const { - return std::visit([this](auto& var) -> bool { return this->IsSubsetOf(var); }, b.Var); + return std::visit([this](auto& var) -> bool { return this->IsSubsetOf(var); }, b.Var); } template <class B> @@ -826,7 +826,7 @@ struct TIntervalSet { } bool IsSubsetOf(const TIntervalSet& y) const { - return std::visit([this](auto& var) -> bool { return this->IsSubsetOf(var); }, y.Var); + return std::visit([this](auto& var) -> bool { return this->IsSubsetOf(var); }, y.Var); } template <class Y> diff --git a/ydb/core/util/operation_queue.h b/ydb/core/util/operation_queue.h index bf70acc29be..845c22c763d 100644 --- a/ydb/core/util/operation_queue.h +++ b/ydb/core/util/operation_queue.h @@ -198,8 +198,8 @@ public: , Timestamp(s) { } - TItemWithTs(const TItemWithTs&) = default; - + TItemWithTs(const TItemWithTs&) = default; + bool operator ==(const TItemWithTs& rhs) const { return Item == rhs.Item; } diff --git a/ydb/core/util/operation_queue_priority_ut.cpp b/ydb/core/util/operation_queue_priority_ut.cpp index 1639ef8c9d7..fd83dc821c8 100644 --- a/ydb/core/util/operation_queue_priority_ut.cpp +++ b/ydb/core/util/operation_queue_priority_ut.cpp @@ -21,8 +21,8 @@ struct TPriorityItem { { } - TPriorityItem(const TPriorityItem&) = default; - + TPriorityItem(const TPriorityItem&) = default; + bool operator ==(const TPriorityItem& rhs) const { // note that only identity intentionally checked return Id == rhs.Id; diff --git a/ydb/core/util/pb.h b/ydb/core/util/pb.h index 81bf964f214..43a656b2d4d 100644 --- a/ydb/core/util/pb.h +++ b/ydb/core/util/pb.h @@ -4,7 +4,7 @@ #include <google/protobuf/text_format.h> #include <util/stream/file.h> #include <util/generic/array_ref.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NKikimr { diff --git a/ydb/core/util/testactorsys.h b/ydb/core/util/testactorsys.h index ce6ad7be10d..31ae3b65d6d 100644 --- a/ydb/core/util/testactorsys.h +++ b/ydb/core/util/testactorsys.h @@ -417,7 +417,7 @@ public: TActorId Register(IActor *actor, const TActorId& parentId = TActorId(), ui32 poolId = 0, std::optional<ui32> hint = std::nullopt, ui32 nodeId = 0) { // count stats - TString name = TypeName(*actor); + TString name = TypeName(*actor); ++ActorStats[name].Created; const bool inserted = ActorName.emplace(actor, std::move(name)).second; Y_VERIFY(inserted); diff --git a/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp b/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp index e484f117be1..a83040b6ea6 100644 --- a/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp +++ b/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp @@ -108,11 +108,11 @@ private: TAsyncStatus RunDataQuery(const TString& query) { auto status = Parent->TableClient->RetryOperation<NYdb::NTable::TDataQueryResult>( - [query](NYdb::NTable::TSession session) { + [query](NYdb::NTable::TSession session) { return session.ExecuteDataQuery( query, NYdb::NTable::TTxControl::BeginTx().CommitTx(), NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::Seconds(10)) - ).Apply([](const auto &future) mutable { + ).Apply([](const auto &future) mutable { return future; }); }); diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp index 398725f7f46..e169e883ec3 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp @@ -1396,7 +1396,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa const auto query = queryBuilder.Build(); auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{}; auto [result, resultSets] = Read(query.Sql, query.Params, requestCounters, debugInfo); - auto prepare = [resultSets=resultSets, resultSetIndex, user, permissions] { + auto prepare = [resultSets=resultSets, resultSetIndex, user, permissions] { if (resultSets->size() != 2) { ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets->size() << ". Please contact internal support"; } diff --git a/ydb/library/yql/dq/opt/dq_opt_build.cpp b/ydb/library/yql/dq/opt/dq_opt_build.cpp index e57b85a9113..ba67f258537 100644 --- a/ydb/library/yql/dq/opt/dq_opt_build.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_build.cpp @@ -289,7 +289,7 @@ public: output = input; TNodeOnNodeOwnedMap replaces; - VisitExpr(input, [&ctx, &replaces](const TExprNode::TPtr& node) { + VisitExpr(input, [&ctx, &replaces](const TExprNode::TPtr& node) { if (node->IsLambda()) { return false; } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp index d2182bbc236..b8925686397 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp @@ -312,7 +312,7 @@ private: const bool SingleShot; struct TFileState { TFileState() - : File(TTempFileHandle::InCurrentDir()) + : File(TTempFileHandle::InCurrentDir()) , TotalSize(0) {} @@ -1582,7 +1582,7 @@ private: const bool SingleShot; struct TFileState { TFileState() - : File(TTempFileHandle::InCurrentDir()) + : File(TTempFileHandle::InCurrentDir()) , TotalSize(0) {} diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h index 0a63ca9d349..4fbfcdf55a0 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h @@ -5,7 +5,7 @@ #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/public/udf/udf_value.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> namespace NKikimr { namespace NMiniKQL { @@ -1030,7 +1030,7 @@ protected: } [[noreturn]] void ThrowNotSupported(const char* func) const { - THROW yexception() << "Unsupported access to '" << func << "' method of: " << TypeName(*this); + THROW yexception() << "Unsupported access to '" << func << "' method of: " << TypeName(*this); } private: diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp index 69788c5c760..743e870ad03 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp @@ -54,7 +54,7 @@ TDataProviderInitializer GetDqDataProviderInitializer( info.Source = CreateDqDataSource(state, execTransformerFactory); info.Sink = CreateDqDataSink(state); - info.OpenSession = [dqGateway, metrics, gatewaysConfig, state]( + info.OpenSession = [dqGateway, metrics, gatewaysConfig, state]( const TString& sessionId, const TString& username, const TOperationProgressWriter& progressWriter, diff --git a/ydb/library/yql/utils/backtrace/backtrace.cpp b/ydb/library/yql/utils/backtrace/backtrace.cpp index f9487f66717..fb1d714bad3 100644 --- a/ydb/library/yql/utils/backtrace/backtrace.cpp +++ b/ydb/library/yql/utils/backtrace/backtrace.cpp @@ -12,7 +12,7 @@ #include <util/stream/format.h> #include <util/stream/output.h> #include <util/system/backtrace.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> #include <util/system/execpath.h> #include <util/system/platform.h> #include <util/system/mlock.h> diff --git a/ydb/services/persqueue_v1/ut/test_utils.h b/ydb/services/persqueue_v1/ut/test_utils.h index c02308b0d7d..297f6b70a5c 100644 --- a/ydb/services/persqueue_v1/ut/test_utils.h +++ b/ydb/services/persqueue_v1/ut/test_utils.h @@ -7,9 +7,9 @@ #include <google/protobuf/message.h> #include <contrib/libs/grpc/include/grpcpp/support/sync_stream.h> #include <util/string/builder.h> -#include <util/system/type_name.h> +#include <util/system/type_name.h> -#define TEST_CASE_NAME (TypeName(*this).rfind("TTestCase") != TString::npos ? TypeName(*this).substr(TypeName(*this).rfind("TTestCase") + 9) : TypeName(*this)) +#define TEST_CASE_NAME (TypeName(*this).rfind("TTestCase") != TString::npos ? TypeName(*this).substr(TypeName(*this).rfind("TTestCase") + 9) : TypeName(*this)) static constexpr int DEBUG_LOG_LEVEL = 7; |