diff options
author | alexvru <alexvru@ydb.tech> | 2023-08-15 21:09:36 +0300 |
---|---|---|
committer | alexvru <alexvru@ydb.tech> | 2023-08-15 21:42:49 +0300 |
commit | d6f67906ea5b369b47bce8e0a7125d66114fdbde (patch) | |
tree | c9c44a3a1a396a6cab33e1260c67f2e5b8b76ea4 | |
parent | f096c967c8a4b645763f901c889ca0335a0e5412 (diff) | |
download | ydb-d6f67906ea5b369b47bce8e0a7125d66114fdbde.tar.gz |
Support BS autoconfig KIKIMR-19031
51 files changed, 1182 insertions, 94 deletions
diff --git a/library/cpp/openssl/CMakeLists.txt b/library/cpp/openssl/CMakeLists.txt index cb9c6e4060..0188258f6c 100644 --- a/library/cpp/openssl/CMakeLists.txt +++ b/library/cpp/openssl/CMakeLists.txt @@ -6,6 +6,8 @@ # original buildsystem will not be accepted. +add_subdirectory(big_integer) +add_subdirectory(crypto) add_subdirectory(holders) add_subdirectory(init) add_subdirectory(io) diff --git a/library/cpp/openssl/big_integer/CMakeLists.darwin-x86_64.txt b/library/cpp/openssl/big_integer/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..64b7f8b472 --- /dev/null +++ b/library/cpp/openssl/big_integer/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-big_integer) +target_link_libraries(cpp-openssl-big_integer PUBLIC + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL +) +target_sources(cpp-openssl-big_integer PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/big_integer/big_integer.cpp +) diff --git a/library/cpp/openssl/big_integer/CMakeLists.linux-aarch64.txt b/library/cpp/openssl/big_integer/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..3e17162a1f --- /dev/null +++ b/library/cpp/openssl/big_integer/CMakeLists.linux-aarch64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-big_integer) +target_link_libraries(cpp-openssl-big_integer PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL +) +target_sources(cpp-openssl-big_integer PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/big_integer/big_integer.cpp +) diff --git a/library/cpp/openssl/big_integer/CMakeLists.linux-x86_64.txt b/library/cpp/openssl/big_integer/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..3e17162a1f --- /dev/null +++ b/library/cpp/openssl/big_integer/CMakeLists.linux-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-big_integer) +target_link_libraries(cpp-openssl-big_integer PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL +) +target_sources(cpp-openssl-big_integer PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/big_integer/big_integer.cpp +) diff --git a/library/cpp/openssl/big_integer/CMakeLists.txt b/library/cpp/openssl/big_integer/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/openssl/big_integer/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/openssl/big_integer/CMakeLists.windows-x86_64.txt b/library/cpp/openssl/big_integer/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..64b7f8b472 --- /dev/null +++ b/library/cpp/openssl/big_integer/CMakeLists.windows-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-big_integer) +target_link_libraries(cpp-openssl-big_integer PUBLIC + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL +) +target_sources(cpp-openssl-big_integer PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/big_integer/big_integer.cpp +) diff --git a/library/cpp/openssl/big_integer/big_integer.cpp b/library/cpp/openssl/big_integer/big_integer.cpp new file mode 100644 index 0000000000..9b6802369a --- /dev/null +++ b/library/cpp/openssl/big_integer/big_integer.cpp @@ -0,0 +1,61 @@ +#include "big_integer.h" + +#include <util/generic/yexception.h> +#include <util/generic/scope.h> +#include <util/stream/output.h> + +#include <openssl/bn.h> + +using namespace NOpenSsl; + +TBigInteger::~TBigInteger() noexcept { + BN_free(Impl_); +} + +TBigInteger TBigInteger::FromULong(ui64 value) { + TBigInteger result(BN_new()); + + Y_ENSURE(result.Impl(), "BN_new() failed"); + Y_ENSURE(BN_set_word(result.Impl(), value) == 1, "BN_set_word() failed"); + + return result; +} + +TBigInteger TBigInteger::FromRegion(const void* ptr, size_t len) { + auto result = BN_bin2bn((ui8*)(ptr), len, nullptr); + + Y_ENSURE(result, "BN_bin2bn() failed"); + + return result; +} + +int TBigInteger::Compare(const TBigInteger& a, const TBigInteger& b) noexcept { + return BN_cmp(a.Impl(), b.Impl()); +} + +size_t TBigInteger::NumBytes() const noexcept { + return BN_num_bytes(Impl_); +} + +size_t TBigInteger::ToRegion(void* to) const noexcept { + const auto ret = BN_bn2bin(Impl_, (unsigned char*)to); + + Y_VERIFY(ret >= 0, "it happens"); + + return ret; +} + +TString TBigInteger::ToDecimalString() const { + auto res = BN_bn2dec(Impl_); + + Y_DEFER { + OPENSSL_free(res); + }; + + return res; +} + +template <> +void Out<TBigInteger>(IOutputStream& out, const TBigInteger& bi) { + out << bi.ToDecimalString(); +} diff --git a/library/cpp/openssl/big_integer/big_integer.h b/library/cpp/openssl/big_integer/big_integer.h new file mode 100644 index 0000000000..07763c5e13 --- /dev/null +++ b/library/cpp/openssl/big_integer/big_integer.h @@ -0,0 +1,57 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> +#include <util/generic/utility.h> +#include <util/generic/string.h> + +struct bignum_st; + +namespace NOpenSsl { + class TBigInteger { + inline TBigInteger(bignum_st* impl) noexcept + : Impl_(impl) + { + } + + static int Compare(const TBigInteger& a, const TBigInteger& b) noexcept; + + public: + inline TBigInteger(TBigInteger&& other) noexcept { + Swap(other); + } + + ~TBigInteger() noexcept; + + static TBigInteger FromULong(ui64 value); + static TBigInteger FromRegion(const void* ptr, size_t len); + + inline const bignum_st* Impl() const noexcept { + return Impl_; + } + + inline bignum_st* Impl() noexcept { + return Impl_; + } + + inline void Swap(TBigInteger& other) noexcept { + DoSwap(Impl_, other.Impl_); + } + + inline friend bool operator==(const TBigInteger& a, const TBigInteger& b) noexcept { + return Compare(a, b) == 0; + } + + inline friend bool operator!=(const TBigInteger& a, const TBigInteger& b) noexcept { + return !(a == b); + } + + size_t NumBytes() const noexcept; + size_t ToRegion(void* to) const noexcept; + + TString ToDecimalString() const; + + private: + bignum_st* Impl_ = nullptr; + }; +} diff --git a/library/cpp/openssl/big_integer/ut/big_integer_ut.cpp b/library/cpp/openssl/big_integer/ut/big_integer_ut.cpp new file mode 100644 index 0000000000..8a0050f531 --- /dev/null +++ b/library/cpp/openssl/big_integer/ut/big_integer_ut.cpp @@ -0,0 +1,43 @@ +#include "big_integer.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/byteorder.h> +#include <util/stream/str.h> + +Y_UNIT_TEST_SUITE(BigInteger) { + using NOpenSsl::TBigInteger; + + Y_UNIT_TEST(Initialization) { + constexpr ui64 testVal = 12345678900; + const auto fromULong = TBigInteger::FromULong(testVal); + + const ui64 testArea = HostToInet(testVal); // transform to big-endian + const auto fromRegion = TBigInteger::FromRegion(&testArea, sizeof(testArea)); + UNIT_ASSERT(fromULong == fromRegion); + UNIT_ASSERT_VALUES_EQUAL(fromULong, fromRegion); + + const auto fromULongOther = TBigInteger::FromULong(22345678900); + UNIT_ASSERT(fromULong != fromULongOther); + } + + Y_UNIT_TEST(Decimal) { + UNIT_ASSERT_VALUES_EQUAL(TBigInteger::FromULong(123456789).ToDecimalString(), "123456789"); + } + + Y_UNIT_TEST(Region) { + const auto v1 = TBigInteger::FromULong(1234567890); + char buf[1024]; + const auto v2 = TBigInteger::FromRegion(buf, v1.ToRegion(buf)); + + UNIT_ASSERT_VALUES_EQUAL(v1, v2); + } + + Y_UNIT_TEST(Output) { + TStringStream ss; + + ss << TBigInteger::FromULong(123456789); + + UNIT_ASSERT_VALUES_EQUAL(ss.Str(), "123456789"); + } +} diff --git a/library/cpp/openssl/big_integer/ut/ya.make b/library/cpp/openssl/big_integer/ut/ya.make new file mode 100644 index 0000000000..473b5c6a9c --- /dev/null +++ b/library/cpp/openssl/big_integer/ut/ya.make @@ -0,0 +1,7 @@ +UNITTEST_FOR(library/cpp/openssl/big_integer) + +SRCS( + big_integer_ut.cpp +) + +END() diff --git a/library/cpp/openssl/big_integer/ya.make b/library/cpp/openssl/big_integer/ya.make new file mode 100644 index 0000000000..5c82a63c6a --- /dev/null +++ b/library/cpp/openssl/big_integer/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +PEERDIR( + contrib/libs/openssl +) + +SRCS( + big_integer.cpp +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/library/cpp/openssl/crypto/CMakeLists.darwin-x86_64.txt b/library/cpp/openssl/crypto/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..d80e96fa54 --- /dev/null +++ b/library/cpp/openssl/crypto/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-crypto) +target_link_libraries(cpp-openssl-crypto PUBLIC + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + cpp-openssl-big_integer + cpp-openssl-init +) +target_sources(cpp-openssl-crypto PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/sha.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/rsa.cpp +) diff --git a/library/cpp/openssl/crypto/CMakeLists.linux-aarch64.txt b/library/cpp/openssl/crypto/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..1465076efc --- /dev/null +++ b/library/cpp/openssl/crypto/CMakeLists.linux-aarch64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-crypto) +target_link_libraries(cpp-openssl-crypto PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + cpp-openssl-big_integer + cpp-openssl-init +) +target_sources(cpp-openssl-crypto PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/sha.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/rsa.cpp +) diff --git a/library/cpp/openssl/crypto/CMakeLists.linux-x86_64.txt b/library/cpp/openssl/crypto/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..1465076efc --- /dev/null +++ b/library/cpp/openssl/crypto/CMakeLists.linux-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-crypto) +target_link_libraries(cpp-openssl-crypto PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + cpp-openssl-big_integer + cpp-openssl-init +) +target_sources(cpp-openssl-crypto PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/sha.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/rsa.cpp +) diff --git a/library/cpp/openssl/crypto/CMakeLists.txt b/library/cpp/openssl/crypto/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/library/cpp/openssl/crypto/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/openssl/crypto/CMakeLists.windows-x86_64.txt b/library/cpp/openssl/crypto/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..d80e96fa54 --- /dev/null +++ b/library/cpp/openssl/crypto/CMakeLists.windows-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +find_package(OpenSSL REQUIRED) + +add_library(cpp-openssl-crypto) +target_link_libraries(cpp-openssl-crypto PUBLIC + contrib-libs-cxxsupp + yutil + OpenSSL::OpenSSL + cpp-openssl-big_integer + cpp-openssl-init +) +target_sources(cpp-openssl-crypto PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/sha.cpp + ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/rsa.cpp +) diff --git a/library/cpp/openssl/crypto/rsa.cpp b/library/cpp/openssl/crypto/rsa.cpp new file mode 100644 index 0000000000..4b1d664826 --- /dev/null +++ b/library/cpp/openssl/crypto/rsa.cpp @@ -0,0 +1,56 @@ +#include "rsa.h" + +#include <library/cpp/openssl/big_integer/big_integer.h> +#include <library/cpp/openssl/init/init.h> + +#include <util/generic/yexception.h> +#include <util/generic/buffer.h> + +#include <openssl/bn.h> +#include <openssl/rsa.h> + +using namespace NOpenSsl; +using namespace NOpenSsl::NRsa; + +namespace { + struct TInit { + inline TInit() { + InitOpenSSL(); + } + } INIT; +} + +TPublicKey::TPublicKey(const TBigInteger& e, const TBigInteger& n) + : Key_(RSA_new()) +{ + Y_ENSURE(Key_, "RSA_new() failed"); + + RSA_set0_key(Key_, BN_dup(n.Impl()), BN_dup(e.Impl()), nullptr); +} + +TPublicKey::~TPublicKey() noexcept { + RSA_free(Key_); +} + +size_t TPublicKey::OutputLength() const noexcept { + return RSA_size(Key_); +} + +size_t TPublicKey::EncryptNoPad(void* dst, const void* src, size_t size) const { + auto len = RSA_public_encrypt(size, (const ui8*)src, (ui8*)dst, Key_, RSA_NO_PADDING); + + Y_ENSURE(len >= 0, "RSA_public_encrypt() failed"); + + return len; +} + +TBigInteger TPublicKey::EncryptNoPad(const TBigInteger& src) const { + const auto len1 = OutputLength(); + const auto len2 = src.NumBytes(); + TBuffer buf(len1 + len2); + + char* buf1 = (char*)buf.Data(); + char* buf2 = buf1 + len1; + + return TBigInteger::FromRegion(buf1, EncryptNoPad(buf1, buf2, src.ToRegion(buf2))); +} diff --git a/library/cpp/openssl/crypto/rsa.h b/library/cpp/openssl/crypto/rsa.h new file mode 100644 index 0000000000..5cb16e195e --- /dev/null +++ b/library/cpp/openssl/crypto/rsa.h @@ -0,0 +1,34 @@ +#pragma once + +#include <util/generic/utility.h> +#include <util/generic/noncopyable.h> + +struct rsa_st; + +namespace NOpenSsl { + class TBigInteger; + + namespace NRsa { + class TPublicKey: public TNonCopyable { + public: + inline TPublicKey(TPublicKey&& other) noexcept { + Swap(other); + } + + TPublicKey(const TBigInteger& e, const TBigInteger& n); + ~TPublicKey() noexcept; + + size_t OutputLength() const noexcept; + + TBigInteger EncryptNoPad(const TBigInteger& src) const; + size_t EncryptNoPad(void* dst, const void* src, size_t size) const; + + inline void Swap(TPublicKey& other) noexcept { + DoSwap(Key_, other.Key_); + } + + private: + rsa_st* Key_ = nullptr; + }; + } +} diff --git a/library/cpp/openssl/crypto/sha.cpp b/library/cpp/openssl/crypto/sha.cpp new file mode 100644 index 0000000000..c142b6635e --- /dev/null +++ b/library/cpp/openssl/crypto/sha.cpp @@ -0,0 +1,62 @@ +#include "sha.h" + +#include <util/generic/yexception.h> + +#include <openssl/sha.h> + +namespace NOpenSsl { + namespace NSha1 { + static_assert(DIGEST_LENGTH == SHA_DIGEST_LENGTH); + + TDigest Calc(const void* data, size_t dataSize) { + TDigest digest; + Y_ENSURE(SHA1(static_cast<const ui8*>(data), dataSize, digest.data()) != nullptr); + return digest; + } + + TCalcer::TCalcer() + : Context{new SHAstate_st} { + Y_ENSURE(SHA1_Init(Context.Get()) == 1); + } + + TCalcer::~TCalcer() { + } + + void TCalcer::Update(const void* data, size_t dataSize) { + Y_ENSURE(SHA1_Update(Context.Get(), data, dataSize) == 1); + } + + TDigest TCalcer::Final() { + TDigest digest; + Y_ENSURE(SHA1_Final(digest.data(), Context.Get()) == 1); + return digest; + } + } + namespace NSha256 { + static_assert(DIGEST_LENGTH == SHA256_DIGEST_LENGTH); + + TDigest Calc(const void* data, size_t dataSize) { + TDigest digest; + Y_ENSURE(SHA256(static_cast<const ui8*>(data), dataSize, digest.data()) != nullptr); + return digest; + } + + TCalcer::TCalcer() + : Context{new SHA256state_st} { + Y_ENSURE(SHA256_Init(Context.Get()) == 1); + } + + TCalcer::~TCalcer() { + } + + void TCalcer::Update(const void* data, size_t dataSize) { + Y_ENSURE(SHA256_Update(Context.Get(), data, dataSize) == 1); + } + + TDigest TCalcer::Final() { + TDigest digest; + Y_ENSURE(SHA256_Final(digest.data(), Context.Get()) == 1); + return digest; + } + } +} diff --git a/library/cpp/openssl/crypto/sha.h b/library/cpp/openssl/crypto/sha.h new file mode 100644 index 0000000000..dbc2dfa526 --- /dev/null +++ b/library/cpp/openssl/crypto/sha.h @@ -0,0 +1,78 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> +#include <util/system/types.h> + +#include <array> + +struct SHAstate_st; +struct SHA256state_st; + +namespace NOpenSsl::NSha1 { + constexpr size_t DIGEST_LENGTH = 20; + using TDigest = std::array<ui8, DIGEST_LENGTH>; + + // not fragmented input + TDigest Calc(const void* data, size_t dataSize); + + inline TDigest Calc(TStringBuf s) { + return Calc(s.data(), s.length()); + } + + // fragmented input + class TCalcer { + public: + TCalcer(); + ~TCalcer(); + void Update(const void* data, size_t dataSize); + + void Update(TStringBuf s) { + Update(s.data(), s.length()); + } + + template <typename T> + void UpdateWithPodValue(const T& value) { + Update(&value, sizeof(value)); + } + + TDigest Final(); + + private: + THolder<SHAstate_st> Context; + }; +} + +namespace NOpenSsl::NSha256 { + constexpr size_t DIGEST_LENGTH = 32; + using TDigest = std::array<ui8, DIGEST_LENGTH>; + + // not fragmented input + TDigest Calc(const void* data, size_t dataSize); + + inline TDigest Calc(TStringBuf s) { + return Calc(s.data(), s.length()); + } + + // fragmented input + class TCalcer { + public: + TCalcer(); + ~TCalcer(); + void Update(const void* data, size_t dataSize); + + void Update(TStringBuf s) { + Update(s.data(), s.length()); + } + + template <typename T> + void UpdateWithPodValue(const T& value) { + Update(&value, sizeof(value)); + } + + TDigest Final(); + + private: + THolder<SHA256state_st> Context; + }; +} diff --git a/library/cpp/openssl/crypto/sha_ut.cpp b/library/cpp/openssl/crypto/sha_ut.cpp new file mode 100644 index 0000000000..d79fa7daad --- /dev/null +++ b/library/cpp/openssl/crypto/sha_ut.cpp @@ -0,0 +1,62 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include "sha.h" + +constexpr TStringBuf SomeAlignedShaTestData = "some _aligned_ test data for SHA-family: align align align align"; + +Y_UNIT_TEST_SUITE(SHA){ + Y_UNIT_TEST(CheckOfTestDataAlignment){ + UNIT_ASSERT_VALUES_EQUAL(SomeAlignedShaTestData.size() % sizeof(ui32), 0); + } + + Y_UNIT_TEST(Sha1Value) { + // bash$ echo -n $SomeAlignedShaTestData | sha1sum + const TStringBuf precalculatedDigest = + "\xA2\x29\x8E\xE2\xEA\x06\x27\x45" + "\x27\xC7\x78\x87\x16\x21\x8A\xA5" + "\x0D\xBA\xBA\xB2"sv; + + auto digest = NOpenSsl::NSha1::Calc(SomeAlignedShaTestData.data(), SomeAlignedShaTestData.size()); + + UNIT_ASSERT_VALUES_EQUAL(precalculatedDigest.size(), digest.size()); + UNIT_ASSERT_VALUES_EQUAL(memcmp(precalculatedDigest.data(), digest.data(), digest.size()), 0); + } + + Y_UNIT_TEST(Sha256Value) { + // bash$ echo -n $SomeAlignedShaTestData | sha256sum + const TStringBuf precalculatedDigest = + "\xED\x64\x0D\x43\xF7\x6D\x71\x98" + "\x39\x19\xF6\xE6\x70\x21\x82\x11" + "\xEF\x3B\xF0\xF4\x35\xBF\x42\xAB" + "\x1C\x5C\x01\xCD\x20\x33\xD2\xFA"sv; + + auto digest = NOpenSsl::NSha256::Calc(SomeAlignedShaTestData.data(), SomeAlignedShaTestData.size()); + + UNIT_ASSERT_VALUES_EQUAL(precalculatedDigest.size(), digest.size()); + UNIT_ASSERT_VALUES_EQUAL(memcmp(precalculatedDigest.data(), digest.data(), digest.size()), 0); + } + + Y_UNIT_TEST(FragmentedEqualNotFragmented) { + const char* head = SomeAlignedShaTestData.data(); + const char* current = head; + NOpenSsl::NSha1::TCalcer sha; + int intValue; + std::copy_n(current, sizeof(intValue), (char*)&intValue); + current += sizeof(intValue); + sha.UpdateWithPodValue(intValue); + double doubleValue; + std::copy_n(current, sizeof(doubleValue), (char*)&doubleValue); + current += sizeof(doubleValue); + sha.UpdateWithPodValue(doubleValue); + char str[7]; + std::copy_n(current, std::size(str), str); + current += std::size(str); + sha.UpdateWithPodValue(str); + sha.Update(current, SomeAlignedShaTestData.size() - (current - head)); + auto fragmentedDigest = sha.Final(); + + auto notFragmentedDigest = NOpenSsl::NSha1::Calc(SomeAlignedShaTestData.data(), SomeAlignedShaTestData.size()); + + UNIT_ASSERT_VALUES_EQUAL(memcmp(fragmentedDigest.data(), notFragmentedDigest.data(), notFragmentedDigest.size()), 0); + } +} // UNITTEST_SIMPLE_SUITE(SHA) diff --git a/library/cpp/openssl/crypto/ut/rsa_ut.cpp b/library/cpp/openssl/crypto/ut/rsa_ut.cpp new file mode 100644 index 0000000000..c11814f8fe --- /dev/null +++ b/library/cpp/openssl/crypto/ut/rsa_ut.cpp @@ -0,0 +1,28 @@ +#include "rsa.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/openssl/big_integer/big_integer.h> + +#include <util/system/byteorder.h> + +using namespace NOpenSsl; +using namespace NOpenSsl::NRsa; + +Y_UNIT_TEST_SUITE(Rsa) { + Y_UNIT_TEST(Encrypt) { + // example from Ru.Wikipedia + const auto originData = TBigInteger::FromULong(111111); + + const auto n = TBigInteger::FromULong(3); + const auto e = TBigInteger::FromULong(9173503); + + // check key reuse + for (size_t i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(TBigInteger::FromULong(4051753), TPublicKey(n, e).EncryptNoPad(originData)); + } + + UNIT_ASSERT_VALUES_EQUAL(originData, TBigInteger::FromULong(111111)); + UNIT_ASSERT_VALUES_EQUAL(n, TBigInteger::FromULong(3)); + UNIT_ASSERT_VALUES_EQUAL(e, TBigInteger::FromULong(9173503)); + } +} diff --git a/library/cpp/openssl/crypto/ut/ya.make b/library/cpp/openssl/crypto/ut/ya.make new file mode 100644 index 0000000000..ffb73352ff --- /dev/null +++ b/library/cpp/openssl/crypto/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/openssl/crypto) + +PEERDIR( + contrib/libs/openssl +) + +SRCS( + rsa_ut.cpp + sha_ut.cpp +) + +END() diff --git a/library/cpp/openssl/crypto/ya.make b/library/cpp/openssl/crypto/ya.make new file mode 100644 index 0000000000..2063e4f125 --- /dev/null +++ b/library/cpp/openssl/crypto/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +PEERDIR( + contrib/libs/openssl + library/cpp/openssl/big_integer + library/cpp/openssl/init +) + +SRCS( + sha.cpp + rsa.cpp +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index d535e642aa..e0aa3e15b6 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -850,6 +850,8 @@ struct TEvBlobStorage { EvNodeConfigPush, EvNodeConfigReversePush, EvNodeConfigUnbind, + EvNodeConfigScatter, + EvNodeConfigGather, // Other EvRunActor = EvPut + 15 * 512, diff --git a/ydb/core/blobstorage/nodewarden/CMakeLists.darwin-x86_64.txt b/ydb/core/blobstorage/nodewarden/CMakeLists.darwin-x86_64.txt index 20b1b8e19e..b22906eea2 100644 --- a/ydb/core/blobstorage/nodewarden/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/blobstorage/nodewarden/CMakeLists.darwin-x86_64.txt @@ -14,6 +14,7 @@ target_link_libraries(core-blobstorage-nodewarden PUBLIC contrib-libs-cxxsupp yutil library-cpp-json + cpp-openssl-crypto ydb-core-base core-blob_depot-agent core-blobstorage-crypto diff --git a/ydb/core/blobstorage/nodewarden/CMakeLists.linux-aarch64.txt b/ydb/core/blobstorage/nodewarden/CMakeLists.linux-aarch64.txt index de99621f47..72d2faa8e3 100644 --- a/ydb/core/blobstorage/nodewarden/CMakeLists.linux-aarch64.txt +++ b/ydb/core/blobstorage/nodewarden/CMakeLists.linux-aarch64.txt @@ -15,6 +15,7 @@ target_link_libraries(core-blobstorage-nodewarden PUBLIC contrib-libs-cxxsupp yutil library-cpp-json + cpp-openssl-crypto ydb-core-base core-blob_depot-agent core-blobstorage-crypto diff --git a/ydb/core/blobstorage/nodewarden/CMakeLists.linux-x86_64.txt b/ydb/core/blobstorage/nodewarden/CMakeLists.linux-x86_64.txt index de99621f47..72d2faa8e3 100644 --- a/ydb/core/blobstorage/nodewarden/CMakeLists.linux-x86_64.txt +++ b/ydb/core/blobstorage/nodewarden/CMakeLists.linux-x86_64.txt @@ -15,6 +15,7 @@ target_link_libraries(core-blobstorage-nodewarden PUBLIC contrib-libs-cxxsupp yutil library-cpp-json + cpp-openssl-crypto ydb-core-base core-blob_depot-agent core-blobstorage-crypto diff --git a/ydb/core/blobstorage/nodewarden/CMakeLists.windows-x86_64.txt b/ydb/core/blobstorage/nodewarden/CMakeLists.windows-x86_64.txt index 20b1b8e19e..b22906eea2 100644 --- a/ydb/core/blobstorage/nodewarden/CMakeLists.windows-x86_64.txt +++ b/ydb/core/blobstorage/nodewarden/CMakeLists.windows-x86_64.txt @@ -14,6 +14,7 @@ target_link_libraries(core-blobstorage-nodewarden PUBLIC contrib-libs-cxxsupp yutil library-cpp-json + cpp-openssl-crypto ydb-core-base core-blob_depot-agent core-blobstorage-crypto diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index c80b263c77..e8aaee7dd3 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -203,7 +203,7 @@ void SetupServices(TTestActorRuntime &runtime, TString extraPath, TIntrusivePtr< static_cast<IPDiskServiceFactory*>(new TStrandedPDiskServiceFactory(runtime)) : static_cast<IPDiskServiceFactory*>(new TRealPDiskServiceFactory()))); // nodeWardenConfig->Monitoring = monitoring; - google::protobuf::TextFormat::ParseFromString(staticConfig, &nodeWardenConfig->ServiceSet); + google::protobuf::TextFormat::ParseFromString(staticConfig, nodeWardenConfig->BlobStorageConfig.MutableServiceSet()); if (nodeIndex == 0) { nodeWardenConfig->SectorMaps[extraPath] = extraSectorMap; @@ -217,7 +217,7 @@ void SetupServices(TTestActorRuntime &runtime, TString extraPath, TIntrusivePtr< TString pDiskPath0 = TStringBuilder() << "SectorMap:" << baseDir << "pdisk_map"; - nodeWardenConfig->ServiceSet.MutablePDisks(0)->SetPath(pDiskPath0); + nodeWardenConfig->BlobStorageConfig.MutableServiceSet()->MutablePDisks(0)->SetPath(pDiskPath0); nodeWardenConfig->SectorMaps[pDiskPath0] = sectorMap; ui64 pDiskGuid = 1; diff --git a/ydb/core/blobstorage/nodewarden/defs.h b/ydb/core/blobstorage/nodewarden/defs.h index d0e2dea33a..217f45bc0e 100644 --- a/ydb/core/blobstorage/nodewarden/defs.h +++ b/ydb/core/blobstorage/nodewarden/defs.h @@ -40,6 +40,7 @@ #include <library/cpp/digest/crc32c/crc32c.h> #include <library/cpp/actors/interconnect/interconnect.h> +#include <library/cpp/openssl/crypto/sha.h> #include <util/folder/dirut.h> #include <util/folder/tempdir.h> diff --git a/ydb/core/blobstorage/nodewarden/node_warden.h b/ydb/core/blobstorage/nodewarden/node_warden.h index fb1cfe370c..3b132ff1a3 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden.h +++ b/ydb/core/blobstorage/nodewarden/node_warden.h @@ -17,7 +17,7 @@ namespace NKikimr { }; struct TNodeWardenConfig : public TThrRefBase { - NKikimrBlobStorage::TNodeWardenServiceSet ServiceSet; + NKikimrConfig::TBlobStorageConfig BlobStorageConfig; TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory; TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds; TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_distributed_config.cpp b/ydb/core/blobstorage/nodewarden/node_warden_distributed_config.cpp index fe9f6565eb..f9f080cd96 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_distributed_config.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_distributed_config.cpp @@ -8,6 +8,7 @@ namespace NKikimr::NStorage { struct TEvPrivate { enum { EvProcessPendingEvent = EventSpaceBegin(TEvents::ES_PRIVATE), + EvQuorumCheckTimeout, }; }; @@ -44,6 +45,7 @@ namespace NKikimr::NStorage { ui64 Cookie; TActorId SessionId; THashSet<ui32> BoundNodeIds; + THashSet<ui64> ScatterTasks; // unanswered scatter queries TBoundNode(ui64 cookie, TActorId sessionId) : Cookie(cookie) @@ -51,6 +53,16 @@ namespace NKikimr::NStorage { {} }; + struct TScatterTask { + THashSet<ui32> PendingNodes; + NKikimrBlobStorage::TEvNodeConfigScatter Task; + std::vector<NKikimrBlobStorage::TEvNodeConfigGather> CollectedReplies; + + TScatterTask(NKikimrBlobStorage::TEvNodeConfigScatter&& task) { + Task.Swap(&task); + } + }; + // current most relevant storage config NKikimrBlobStorage::TStorageConfig StorageConfig; @@ -58,24 +70,50 @@ namespace NKikimr::NStorage { std::optional<TBinding> Binding; ui64 BindingCookie = RandomNumber<ui64>(); TBindQueue BindQueue; - ui32 NumPeerNodes = 0; bool Scheduled = false; // incoming bindings THashMap<ui32, TBoundNode> DirectBoundNodes; // a set of nodes directly bound to this one THashMap<ui32, ui32> AllBoundNodes; // counter may be more than 2 in case of races, but not for long + // pending event queue std::deque<TAutoPtr<IEventHandle>> PendingEvents; std::vector<ui32> NodeIds; + // scatter tasks + ui64 NextScatterCookie = RandomNumber<ui64>(); + THashMap<ui64, TScatterTask> ScatterTasks; + + // root node operation + enum class ERootState { + INITIAL, + QUORUM_CHECK_TIMEOUT, + COLLECT_CONFIG, + }; + static constexpr TDuration QuorumCheckTimeout = TDuration::Seconds(1); // time to wait after obtaining quorum + ERootState RootState = ERootState::INITIAL; + public: void Bootstrap() { STLOG(PRI_DEBUG, BS_NODE, NWDC00, "Bootstrap"); - StorageConfig.SetGeneration(1); + StorageConfig.SetFingerprint(CalculateFingerprint(StorageConfig)); Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes(true)); Become(&TThis::StateWaitForList); } + TString CalculateFingerprint(const NKikimrBlobStorage::TStorageConfig& config) { + NKikimrBlobStorage::TStorageConfig temp; + temp.CopyFrom(config); + temp.ClearFingerprint(); + + TString s; + const bool success = temp.SerializeToString(&s); + Y_VERIFY(success); + + auto digest = NOpenSsl::NSha1::Calc(s.data(), s.size()); + return TString(reinterpret_cast<const char*>(digest.data()), digest.size()); + } + void Handle(TEvInterconnect::TEvNodesInfo::TPtr ev) { STLOG(PRI_DEBUG, BS_NODE, NWDC11, "TEvNodesInfo"); @@ -110,6 +148,7 @@ namespace NKikimr::NStorage { UnbindNode(nodeId, true); if (Binding && Binding->NodeId == nodeId) { Binding.reset(); + AbortAllScatterTasks(); bindingReset = true; } changes = true; @@ -127,13 +166,11 @@ namespace NKikimr::NStorage { // issue updates NodeIds = std::move(nodeIds); BindQueue.Update(NodeIds); - NumPeerNodes = NodeIds.size() - 1; IssueNextBindRequest(); if (bindingReset) { for (const auto& [nodeId, info] : DirectBoundNodes) { - SendEvent(nodeId, info.Cookie, info.SessionId, std::make_unique<TEvNodeConfigReversePush>(nullptr, - GetRootNodeId())); + SendEvent(nodeId, info.Cookie, info.SessionId, std::make_unique<TEvNodeConfigReversePush>(GetRootNodeId())); } } } @@ -152,7 +189,8 @@ namespace NKikimr::NStorage { // Binding to peer nodes void IssueNextBindRequest() { - if (!Binding && AllBoundNodes.size() + 1 /* including this one */ < NumPeerNodes) { + CheckRootNodeStatus(); + if (!Binding && AllBoundNodes.size() < NodeIds.size() && RootState == ERootState::INITIAL) { const TMonotonic now = TActivationContext::Monotonic(); TMonotonic closest; if (std::optional<ui32> nodeId = BindQueue.Pick(now, &closest)) { @@ -182,8 +220,7 @@ namespace NKikimr::NStorage { } STLOG(PRI_DEBUG, BS_NODE, NWDC09, "Continuing bind", (Binding, Binding)); - SendEvent(nodeId, Binding->Cookie, Binding->SessionId, std::make_unique<TEvNodeConfigPush>(&StorageConfig, - AllBoundNodes)); + SendEvent(nodeId, Binding->Cookie, Binding->SessionId, std::make_unique<TEvNodeConfigPush>(AllBoundNodes)); } } @@ -239,21 +276,10 @@ namespace NKikimr::NStorage { UnsubscribeInterconnect(senderNodeId, sessionId); } - // check if we have newer configuration from the peer - bool configUpdated = false; - if (record.HasStorageConfig()) { - const auto& config = record.GetStorageConfig(); - if (StorageConfig.GetGeneration() < config.GetGeneration()) { - StorageConfig.Swap(record.MutableStorageConfig()); - configUpdated = true; - } - } - // fan-out updates to the following peers - if (configUpdated || rootUpdated) { + if (rootUpdated) { for (const auto& [nodeId, info] : DirectBoundNodes) { - SendEvent(nodeId, info, std::make_unique<TEvNodeConfigReversePush>( - configUpdated ? &StorageConfig : nullptr, GetRootNodeId())); + SendEvent(nodeId, info, std::make_unique<TEvNodeConfigReversePush>(GetRootNodeId())); } } } else { @@ -319,15 +345,14 @@ namespace NKikimr::NStorage { } } - // prepare configuration push down - auto downEv = Binding - ? std::make_unique<TEvNodeConfigPush>() - : nullptr; - - // and configuration push up - auto upEv = record.GetInitial() - ? std::make_unique<TEvNodeConfigReversePush>(nullptr, GetRootNodeId()) - : nullptr; + // configuration push message + std::unique_ptr<TEvNodeConfigPush> pushEv; + auto getPushEv = [&] { + if (Binding && !pushEv) { + pushEv = std::make_unique<TEvNodeConfigPush>(); + } + return pushEv.get(); + }; // insert new connection into map (if there is none) const auto [it, inserted] = DirectBoundNodes.try_emplace(senderNodeId, ev->Cookie, ev->InterconnectSession); @@ -345,7 +370,7 @@ namespace NKikimr::NStorage { } // account newly bound node itself and add it to the record - AddBound(senderNodeId, downEv.get()); + AddBound(senderNodeId, getPushEv()); } else if (ev->Cookie != info.Cookie || ev->InterconnectSession != info.SessionId) { STLOG(PRI_CRIT, BS_NODE, NWDC12, "distributed configuration protocol violation: cookie/session mismatch", (Sender, ev->Sender), @@ -361,7 +386,7 @@ namespace NKikimr::NStorage { // process added items for (const ui32 nodeId : record.GetNewBoundNodeIds()) { if (info.BoundNodeIds.insert(nodeId).second) { - AddBound(nodeId, downEv.get()); + AddBound(nodeId, getPushEv()); } else { STLOG(PRI_CRIT, BS_NODE, NWDC04, "distributed configuration protocol violation: adding duplicate item", (Sender, ev->Sender), @@ -377,7 +402,7 @@ namespace NKikimr::NStorage { // process deleted items for (const ui32 nodeId : record.GetDeletedBoundNodeIds()) { if (info.BoundNodeIds.erase(nodeId)) { - DeleteBound(nodeId, downEv.get()); + DeleteBound(nodeId, getPushEv()); } else { STLOG(PRI_CRIT, BS_NODE, NWDC05, "distributed configuration protocol violation: deleting nonexisting item", (Sender, ev->Sender), @@ -390,31 +415,12 @@ namespace NKikimr::NStorage { } } - // process configuration update - if (record.HasStorageConfig()) { - const auto& config = record.GetStorageConfig(); - if (StorageConfig.GetGeneration() < config.GetGeneration()) { - StorageConfig.Swap(record.MutableStorageConfig()); - if (downEv) { - downEv->Record.MutableStorageConfig()->CopyFrom(StorageConfig); - } - - for (const auto& [nodeId, info] : DirectBoundNodes) { - if (nodeId != senderNodeId) { - SendEvent(nodeId, info, std::make_unique<TEvNodeConfigReversePush>(&StorageConfig, GetRootNodeId())); - } - } - } else if (config.GetGeneration() < StorageConfig.GetGeneration() && upEv) { - upEv->Record.MutableStorageConfig()->CopyFrom(StorageConfig); - } + if (pushEv) { + SendEvent(*Binding, std::move(pushEv)); } - if (downEv && (downEv->Record.HasStorageConfig() || downEv->Record.NewBoundNodeIdsSize() || - downEv->Record.DeletedBoundNodeIdsSize())) { - SendEvent(*Binding, std::move(downEv)); - } - if (upEv) { - SendEvent(senderNodeId, info, std::move(upEv)); + if (!Binding) { + CheckRootNodeStatus(); } } @@ -453,6 +459,11 @@ namespace NKikimr::NStorage { SendEvent(*Binding, std::move(ev)); } + // abort all unprocessed scatter tasks + for (const ui64 cookie : info.ScatterTasks) { + AbortScatterTask(cookie, nodeId); + } + const TActorId sessionId = info.SessionId; DirectBoundNodes.erase(it); @@ -469,6 +480,195 @@ namespace NKikimr::NStorage { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Root node operation + + void CheckRootNodeStatus() { + if (RootState == ERootState::INITIAL && !Binding && HasQuorum()) { + STLOG(PRI_DEBUG, BS_NODE, NWDC18, "Starting QUORUM_CHECK_TIMEOUT"); + TActivationContext::Schedule(QuorumCheckTimeout, new IEventHandle(TEvPrivate::EvQuorumCheckTimeout, 0, + SelfId(), {}, nullptr, 0)); + RootState = ERootState::QUORUM_CHECK_TIMEOUT; + } + } + + void HandleQuorumCheckTimeout() { + if (HasQuorum()) { + STLOG(PRI_DEBUG, BS_NODE, NWDC19, "Quorum check timeout hit, quorum remains"); + + RootState = ERootState::COLLECT_CONFIG; + + NKikimrBlobStorage::TEvNodeConfigScatter task; + task.MutableCollectConfigs(); + IssueScatterTask(std::move(task)); + } else { + STLOG(PRI_DEBUG, BS_NODE, NWDC20, "Quorum check timeout hit, quorum reset"); + RootState = ERootState::INITIAL; // fall back to waiting for quorum + IssueNextBindRequest(); + } + } + + void ProcessGather(NKikimrBlobStorage::TEvNodeConfigGather&& res) { + switch (RootState) { + case ERootState::COLLECT_CONFIG: + STLOG(PRI_DEBUG, BS_NODE, NWDC27, "ProcessGather(COLLECT_CONFIG)", (Res, res)); + break; + + default: + break; + } + } + + bool HasQuorum() const { + // we have strict majority of all nodes (including this one) + return AllBoundNodes.size() + 1 > (NodeIds.size() + 1) / 2; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Scatter/gather logic + + void IssueScatterTask(NKikimrBlobStorage::TEvNodeConfigScatter&& task) { + const ui64 cookie = NextScatterCookie++; + STLOG(PRI_DEBUG, BS_NODE, NWDC21, "IssueScatterTask", (Task, task), (Cookie, cookie)); + const auto [it, inserted] = ScatterTasks.try_emplace(cookie, std::move(task)); + Y_VERIFY(inserted); + TScatterTask& scatterTask = it->second; + for (auto& [nodeId, info] : DirectBoundNodes) { + auto ev = std::make_unique<TEvNodeConfigScatter>(); + ev->Record.CopyFrom(scatterTask.Task); + ev->Record.SetCookie(cookie); + SendEvent(nodeId, info, std::move(ev)); + info.ScatterTasks.insert(cookie); + scatterTask.PendingNodes.insert(nodeId); + } + if (scatterTask.PendingNodes.empty()) { + CompleteScatterTask(scatterTask); + ScatterTasks.erase(it); + } + } + + void CompleteScatterTask(TScatterTask& task) { + STLOG(PRI_DEBUG, BS_NODE, NWDC22, "CompleteScatterTask", (Task, task.Task)); + + NKikimrBlobStorage::TEvNodeConfigGather res; + if (task.Task.HasCookie()) { + res.SetCookie(task.Task.GetCookie()); + } + + switch (task.Task.GetRequestCase()) { + case NKikimrBlobStorage::TEvNodeConfigScatter::kCollectConfigs: + GenerateCollectConfigs(res.MutableCollectConfigs(), task); + break; + + case NKikimrBlobStorage::TEvNodeConfigScatter::kApplyConfigs: + break; + + case NKikimrBlobStorage::TEvNodeConfigScatter::REQUEST_NOT_SET: + // unexpected case + break; + } + + if (Binding && res.HasCookie()) { + auto reply = std::make_unique<TEvNodeConfigGather>(); + reply->Record.CopyFrom(res); + SendEvent(*Binding, std::move(reply)); + } else if (!res.HasCookie()) { + ProcessGather(std::move(res)); + } + } + + void GenerateCollectConfigs(NKikimrBlobStorage::TEvNodeConfigGather::TCollectConfigs *response, TScatterTask& task) { + THashMap<std::tuple<ui64, TString>, NKikimrBlobStorage::TEvNodeConfigGather::TCollectConfigs::TItem*> configs; + + auto addConfig = [&](const NKikimrBlobStorage::TStorageConfig& config, const auto& nodeIds) { + const auto key = std::make_tuple(config.GetGeneration(), config.GetFingerprint()); + auto& ptr = configs[key]; + if (!ptr) { + ptr = response->AddItems(); + ptr->MutableConfig()->CopyFrom(config); + } + for (const ui32 nodeId : nodeIds) { + ptr->AddNodeIds(nodeId); + } + }; + + addConfig(StorageConfig, std::initializer_list<ui32>{SelfId().NodeId()}); + + for (const auto& reply : task.CollectedReplies) { + if (reply.HasCollectConfigs()) { + for (const auto& item : reply.GetCollectConfigs().GetItems()) { + addConfig(item.GetConfig(), item.GetNodeIds()); + } + } + } + } + + void AbortScatterTask(ui64 cookie, ui32 nodeId) { + STLOG(PRI_DEBUG, BS_NODE, NWDC23, "AbortScatterTask", (Cookie, cookie), (NodeId, nodeId)); + + const auto it = ScatterTasks.find(cookie); + Y_VERIFY(it != ScatterTasks.end()); + TScatterTask& task = it->second; + + const size_t n = task.PendingNodes.erase(nodeId); + Y_VERIFY(n == 1); + if (task.PendingNodes.empty()) { + CompleteScatterTask(task); + ScatterTasks.erase(it); + } + } + + void AbortAllScatterTasks() { + STLOG(PRI_DEBUG, BS_NODE, NWDC24, "AbortAllScatterTasks"); + + for (auto& [cookie, task] : std::exchange(ScatterTasks, {})) { + for (const ui32 nodeId : task.PendingNodes) { + const auto it = DirectBoundNodes.find(nodeId); + Y_VERIFY(it != DirectBoundNodes.end()); + TBoundNode& info = it->second; + const size_t n = info.ScatterTasks.erase(cookie); + Y_VERIFY(n == 1); + } + } + } + + void Handle(TEvNodeConfigScatter::TPtr ev) { + STLOG(PRI_DEBUG, BS_NODE, NWDC25, "TEvNodeConfigScatter", (Binding, Binding), (Sender, ev->Sender), + (Cookie, ev->Cookie), (SessionId, ev->InterconnectSession), (Record, ev->Get()->Record)); + + if (Binding && Binding->Expected(*ev)) { + IssueScatterTask(std::move(ev->Get()->Record)); + } + } + + void Handle(TEvNodeConfigGather::TPtr ev) { + STLOG(PRI_DEBUG, BS_NODE, NWDC26, "TEvNodeConfigGather", (Sender, ev->Sender), (Cookie, ev->Cookie), + (SessionId, ev->InterconnectSession), (Record, ev->Get()->Record)); + + const ui32 senderNodeId = ev->Sender.NodeId(); + if (const auto it = DirectBoundNodes.find(senderNodeId); it != DirectBoundNodes.end() + && ev->Cookie == it->second.Cookie + && ev->InterconnectSession == it->second.SessionId) { + TBoundNode& info = it->second; + auto& record = ev->Get()->Record; + if (const auto jt = ScatterTasks.find(record.GetCookie()); jt != ScatterTasks.end()) { + const size_t n = info.ScatterTasks.erase(jt->first); + Y_VERIFY(n == 1); + + TScatterTask& task = jt->second; + record.Swap(&task.CollectedReplies.emplace_back()); + const size_t m = task.PendingNodes.erase(senderNodeId); + Y_VERIFY(m == 1); + if (task.PendingNodes.empty()) { + CompleteScatterTask(task); + ScatterTasks.erase(jt); + } + } else { + Y_VERIFY_DEBUG(false); + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Event delivery void SendEvent(ui32 nodeId, ui64 cookie, TActorId sessionId, std::unique_ptr<IEventBase> ev) { @@ -506,11 +706,11 @@ namespace NKikimr::NStorage { STLOG(PRI_DEBUG, BS_NODE, NWDC10, "Binding aborted by disconnection", (Binding, Binding)); Binding.reset(); + AbortAllScatterTasks(); IssueNextBindRequest(); for (const auto& [nodeId, info] : DirectBoundNodes) { - SendEvent(nodeId, info.Cookie, info.SessionId, std::make_unique<TEvNodeConfigReversePush>(nullptr, - GetRootNodeId())); + SendEvent(nodeId, info.Cookie, info.SessionId, std::make_unique<TEvNodeConfigReversePush>(GetRootNodeId())); } } } @@ -535,6 +735,24 @@ namespace NKikimr::NStorage { if (Binding) { Y_VERIFY(std::binary_search(NodeIds.begin(), NodeIds.end(), Binding->NodeId)); } + + for (const auto& [cookie, task] : ScatterTasks) { + for (const ui32 nodeId : task.PendingNodes) { + const auto it = DirectBoundNodes.find(nodeId); + Y_VERIFY(it != DirectBoundNodes.end()); + TBoundNode& info = it->second; + Y_VERIFY(info.ScatterTasks.contains(cookie)); + } + } + + for (const auto& [nodeId, info] : DirectBoundNodes) { + for (const ui64 cookie : info.ScatterTasks) { + const auto it = ScatterTasks.find(cookie); + Y_VERIFY(it != ScatterTasks.end()); + TScatterTask& task = it->second; + Y_VERIFY(task.PendingNodes.contains(nodeId)); + } + } #endif } @@ -565,9 +783,12 @@ namespace NKikimr::NStorage { hFunc(TEvNodeConfigPush, Handle); hFunc(TEvNodeConfigReversePush, Handle); hFunc(TEvNodeConfigUnbind, Handle); + hFunc(TEvNodeConfigScatter, Handle); + hFunc(TEvNodeConfigGather, Handle); hFunc(TEvInterconnect::TEvNodesInfo, Handle); hFunc(TEvInterconnect::TEvNodeConnected, Handle); hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); + cFunc(TEvPrivate::EvQuorumCheckTimeout, HandleQuorumCheckTimeout); cFunc(TEvents::TSystem::Wakeup, HandleWakeup); cFunc(TEvents::TSystem::Poison, PassAway); ) diff --git a/ydb/core/blobstorage/nodewarden/node_warden_events.h b/ydb/core/blobstorage/nodewarden/node_warden_events.h index 7e7873f10a..9f54554073 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_events.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_events.h @@ -12,10 +12,7 @@ namespace NKikimr::NStorage { TEvNodeConfigPush() = default; // ctor for initial push request - TEvNodeConfigPush(const NKikimrBlobStorage::TStorageConfig *config, const THashMap<ui32, ui32>& boundNodeIds) { - if (config) { - Record.MutableStorageConfig()->CopyFrom(*config); - } + TEvNodeConfigPush(const THashMap<ui32, ui32>& boundNodeIds) { for (const auto [nodeId, counter] : boundNodeIds) { Record.AddNewBoundNodeIds(nodeId); } @@ -28,10 +25,7 @@ namespace NKikimr::NStorage { { TEvNodeConfigReversePush() = default; - TEvNodeConfigReversePush(const NKikimrBlobStorage::TStorageConfig *config, ui32 rootNodeId) { - if (config) { - Record.MutableStorageConfig()->CopyFrom(*config); - } + TEvNodeConfigReversePush(ui32 rootNodeId) { Record.SetRootNodeId(rootNodeId); } @@ -44,7 +38,16 @@ namespace NKikimr::NStorage { struct TEvNodeConfigUnbind : TEventPB<TEvNodeConfigUnbind, NKikimrBlobStorage::TEvNodeConfigUnbind, TEvBlobStorage::EvNodeConfigUnbind> + {}; + + struct TEvNodeConfigScatter + : TEventPB<TEvNodeConfigScatter, NKikimrBlobStorage::TEvNodeConfigScatter, TEvBlobStorage::EvNodeConfigScatter> { + TEvNodeConfigScatter() = default; }; + struct TEvNodeConfigGather + : TEventPB<TEvNodeConfigGather, NKikimrBlobStorage::TEvNodeConfigGather, TEvBlobStorage::EvNodeConfigGather> + {}; + } // NKikimr::NStorage diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index 174b3114a7..eb0edf6a5c 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -90,7 +90,7 @@ void TNodeWarden::Bootstrap() { } // start replication broker - const auto& replBrokerConfig = Cfg->ServiceSet.GetReplBrokerConfig(); + const auto& replBrokerConfig = Cfg->BlobStorageConfig.GetServiceSet().GetReplBrokerConfig(); ui64 requestBytesPerSecond = 500000000; // 500 MB/s by default if (replBrokerConfig.HasTotalRequestBytesPerSecond()) { @@ -110,11 +110,13 @@ void TNodeWarden::Bootstrap() { actorSystem->RegisterLocalService(MakeBlobStorageReplBrokerID(), Register(CreateReplBrokerActor(maxBytes))); // determine if we are running in 'mock' mode - EnableProxyMock = Cfg->ServiceSet.GetEnableProxyMock(); + EnableProxyMock = Cfg->BlobStorageConfig.GetServiceSet().GetEnableProxyMock(); // Start a statically configured set - ApplyServiceSet(Cfg->ServiceSet, true, false, false); - StartStaticProxies(); + if (Cfg->BlobStorageConfig.HasServiceSet()) { + ApplyServiceSet(Cfg->BlobStorageConfig.GetServiceSet(), true, false, false); + StartStaticProxies(); + } EstablishPipe(); Send(GetNameserviceActorId(), new TEvInterconnect::TEvGetNode(LocalNodeId)); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 6d47f1e5ea..ec56a334f4 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -124,9 +124,9 @@ namespace NKikimr::NStorage { , EnablePutBatching(Cfg->FeatureFlags.GetEnablePutBatchingForBlobStorage(), false, true) , EnableVPatch(Cfg->FeatureFlags.GetEnableVPatch(), false, true) { - Y_VERIFY(Cfg->ServiceSet.AvailabilityDomainsSize() <= 1); + Y_VERIFY(Cfg->BlobStorageConfig.GetServiceSet().AvailabilityDomainsSize() <= 1); AvailDomainId = 1; - for (const auto& domain : Cfg->ServiceSet.GetAvailabilityDomains()) { + for (const auto& domain : Cfg->BlobStorageConfig.GetServiceSet().GetAvailabilityDomains()) { AvailDomainId = domain; } } @@ -527,6 +527,8 @@ namespace NKikimr::NStorage { fFunc(TEvBlobStorage::EvNodeConfigPush, ForwardToDistributedConfigKeeper); fFunc(TEvBlobStorage::EvNodeConfigReversePush, ForwardToDistributedConfigKeeper); fFunc(TEvBlobStorage::EvNodeConfigUnbind, ForwardToDistributedConfigKeeper); + fFunc(TEvBlobStorage::EvNodeConfigScatter, ForwardToDistributedConfigKeeper); + fFunc(TEvBlobStorage::EvNodeConfigGather, ForwardToDistributedConfigKeeper); default: EnqueuePendingMessage(ev); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp index 2bc1613f7c..8f643f0ae0 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp @@ -137,7 +137,7 @@ namespace NKikimr::NStorage { TPDiskCategory category(record.Record.GetPDiskCategory()); std::optional<ui64> readBytesPerSecond, writeBytesPerSecond; - for (const auto& item : Cfg->ServiceSet.GetReplBrokerConfig().GetMediaTypeQuota()) { + for (const auto& item : Cfg->BlobStorageConfig.GetServiceSet().GetReplBrokerConfig().GetMediaTypeQuota()) { if (PDiskTypeToPDiskType(item.GetType()) == category.Type()) { if (item.HasReadBytesPerSecond()) { readBytesPerSecond.emplace(item.GetReadBytesPerSecond()); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp b/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp index 0a3c507d87..9dea1ff94c 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp @@ -75,7 +75,8 @@ void TNodeWarden::StartVirtualGroupAgent(ui32 groupId) { } void TNodeWarden::StartStaticProxies() { - for (const auto& group : Cfg->ServiceSet.GetGroups()) { + Y_VERIFY(Cfg->BlobStorageConfig.HasServiceSet()); + for (const auto& group : Cfg->BlobStorageConfig.GetServiceSet().GetGroups()) { StartLocalProxy(group.GetGroupID()); } } diff --git a/ydb/core/blobstorage/nodewarden/ut_sequence/dsproxy_config_retrieval.cpp b/ydb/core/blobstorage/nodewarden/ut_sequence/dsproxy_config_retrieval.cpp index 16648d78cb..17a2ac4f84 100644 --- a/ydb/core/blobstorage/nodewarden/ut_sequence/dsproxy_config_retrieval.cpp +++ b/ydb/core/blobstorage/nodewarden/ut_sequence/dsproxy_config_retrieval.cpp @@ -71,7 +71,7 @@ void SetupServices(TTestBasicRuntime& runtime) { SetupStateStorage(runtime, i); auto config = MakeIntrusive<TNodeWardenConfig>(new TStrandedPDiskServiceFactory(runtime)); config->SectorMaps[path] = sectorMap; - config->ServiceSet = configs[i]; + config->BlobStorageConfig.MutableServiceSet()->CopyFrom(configs[i]); SetupBSNodeWarden(runtime, i, config); SetupTabletResolver(runtime, i); SetupNodeWhiteboard(runtime, i); diff --git a/ydb/core/blobstorage/nodewarden/ya.make b/ydb/core/blobstorage/nodewarden/ya.make index 052434cf13..5a14cbb471 100644 --- a/ydb/core/blobstorage/nodewarden/ya.make +++ b/ydb/core/blobstorage/nodewarden/ya.make @@ -24,6 +24,7 @@ SRCS( PEERDIR( library/cpp/json + library/cpp/openssl/crypto ydb/core/base ydb/core/blob_depot/agent ydb/core/blobstorage/crypto diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h index 74c3d2a372..8eb4afab65 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h @@ -290,7 +290,7 @@ struct TEnvironmentSetup { warden.reset(new TNodeWardenMockActor(Settings.NodeWardenMockSetup)); } else { auto config = MakeIntrusive<TNodeWardenConfig>(new TMockPDiskServiceFactory(*this)); - config->ServiceSet.AddAvailabilityDomains(DomainId); + config->BlobStorageConfig.MutableServiceSet()->AddAvailabilityDomains(DomainId); config->VDiskReplPausedAtStart = Settings.VDiskReplPausedAtStart; if (Settings.ConfigPreprocessor) { Settings.ConfigPreprocessor(nodeId, *config); diff --git a/ydb/core/blobstorage/ut_testshard/env.h b/ydb/core/blobstorage/ut_testshard/env.h index a3dc4fe8fc..bd55d1f734 100644 --- a/ydb/core/blobstorage/ut_testshard/env.h +++ b/ydb/core/blobstorage/ut_testshard/env.h @@ -214,7 +214,7 @@ struct TEnvironmentSetup { } auto config = MakeIntrusive<TNodeWardenConfig>(new TMockPDiskServiceFactory(*this)); - config->ServiceSet.AddAvailabilityDomains(DomainId); + config->BlobStorageConfig.MutableServiceSet()->AddAvailabilityDomains(DomainId); std::unique_ptr<IActor> warden(CreateBSNodeWarden(config)); const TActorId wardenId = Runtime->Register(warden.release(), nodeId); diff --git a/ydb/core/cms/cms_ut_common.cpp b/ydb/core/cms/cms_ut_common.cpp index 5bfea71ce2..e30891e2f5 100644 --- a/ydb/core/cms/cms_ut_common.cpp +++ b/ydb/core/cms/cms_ut_common.cpp @@ -441,7 +441,7 @@ static void SetupServices(TTestActorRuntime &runtime, new TNodeWardenConfig(STRAND_PDISK && !runtime.IsRealThreads() ? static_cast<IPDiskServiceFactory*>(new TStrandedPDiskServiceFactory(runtime)) : static_cast<IPDiskServiceFactory*>(new TRealPDiskServiceFactory())); - google::protobuf::TextFormat::ParseFromString(staticConfig, &nodeWardenConfig->ServiceSet); + google::protobuf::TextFormat::ParseFromString(staticConfig, nodeWardenConfig->BlobStorageConfig.MutableServiceSet()); if (nodeIndex == 0) { TString pDiskPath; @@ -461,7 +461,7 @@ static void SetupServices(TTestActorRuntime &runtime, static TTempDir tempDir; pDiskPath = tempDir() + "/pdisk0.dat"; } - nodeWardenConfig->ServiceSet.MutablePDisks(0)->SetPath(pDiskPath); + nodeWardenConfig->BlobStorageConfig.MutableServiceSet()->MutablePDisks(0)->SetPath(pDiskPath); ui64 pDiskGuid = 1; static ui64 iteration = 0; ++iteration; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 520262927b..172775dac7 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -914,7 +914,7 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se const auto& bsc = Config.GetBlobStorageConfig(); appData->StaticBlobStorageConfig->MergeFrom(bsc.GetServiceSet()); nodeWardenConfig->FeatureFlags = Config.GetFeatureFlags(); - nodeWardenConfig->ServiceSet.MergeFrom(bsc.GetServiceSet()); + nodeWardenConfig->BlobStorageConfig.CopyFrom(bsc); if (Config.HasVDiskConfig()) { nodeWardenConfig->AllVDiskKinds->Merge(Config.GetVDiskConfig()); } diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index c155a47e51..5861ffbbe0 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -159,7 +159,7 @@ namespace { STRAND_PDISK && !runtime.IsRealThreads() ? static_cast<IPDiskServiceFactory*>(new TStrandedPDiskServiceFactory(runtime)) : static_cast<IPDiskServiceFactory*>(new TRealPDiskServiceFactory())); //nodeWardenConfig->Monitoring = monitoring; - google::protobuf::TextFormat::ParseFromString(staticConfig, &nodeWardenConfig->ServiceSet); + google::protobuf::TextFormat::ParseFromString(staticConfig, nodeWardenConfig->BlobStorageConfig.MutableServiceSet()); TIntrusivePtr<TNodeWardenConfig> existingNodeWardenConfig = NodeWardenConfigs[nodeIndex]; if (existingNodeWardenConfig != nullptr) { @@ -193,7 +193,7 @@ namespace { static TTempDir tempDir; pDiskPath = tempDir() + "/pdisk.dat"; } - nodeWardenConfig->ServiceSet.MutablePDisks(0)->SetPath(pDiskPath); + nodeWardenConfig->BlobStorageConfig.MutableServiceSet()->MutablePDisks(0)->SetPath(pDiskPath); ui64 pDiskGuid = 1; static ui64 iteration = 0; ++iteration; @@ -283,7 +283,7 @@ namespace { for (ui32 nodeIndex = 0; nodeIndex < runtime.GetNodeCount(); ++nodeIndex) { auto it = NodeWardenConfigs.find(nodeIndex); if (it != NodeWardenConfigs.end()) { - runtime.GetAppData(nodeIndex).StaticBlobStorageConfig = MakeHolder<NKikimrBlobStorage::TNodeWardenServiceSet>(it->second->ServiceSet); + runtime.GetAppData(nodeIndex).StaticBlobStorageConfig = MakeHolder<NKikimrBlobStorage::TNodeWardenServiceSet>(it->second->BlobStorageConfig.GetServiceSet()); } } diff --git a/ydb/core/mind/node_broker_ut.cpp b/ydb/core/mind/node_broker_ut.cpp index c5a1be3371..49f2acca65 100644 --- a/ydb/core/mind/node_broker_ut.cpp +++ b/ydb/core/mind/node_broker_ut.cpp @@ -92,7 +92,7 @@ void SetupServices(TTestActorRuntime &runtime, new TNodeWardenConfig(STRAND_PDISK && !runtime.IsRealThreads() ? static_cast<IPDiskServiceFactory*>(new TStrandedPDiskServiceFactory(runtime)) : static_cast<IPDiskServiceFactory*>(new TRealPDiskServiceFactory())); - google::protobuf::TextFormat::ParseFromString(staticConfig, &nodeWardenConfig->ServiceSet); + google::protobuf::TextFormat::ParseFromString(staticConfig, nodeWardenConfig->BlobStorageConfig.MutableServiceSet()); TIntrusivePtr<TNodeWardenConfig> existingNodeWardenConfig = NodeWardenConfigs[nodeIndex]; if (existingNodeWardenConfig != nullptr) { @@ -117,7 +117,7 @@ void SetupServices(TTestActorRuntime &runtime, static TTempDir tempDir; pDiskPath = tempDir() + "/pdisk0.dat"; } - nodeWardenConfig->ServiceSet.MutablePDisks(0)->SetPath(pDiskPath); + nodeWardenConfig->BlobStorageConfig.MutableServiceSet()->MutablePDisks(0)->SetPath(pDiskPath); ui64 pDiskGuid = 1; static ui64 iteration = 0; ++iteration; diff --git a/ydb/core/mind/ut_fat/blobstorage_node_warden_ut_fat.cpp b/ydb/core/mind/ut_fat/blobstorage_node_warden_ut_fat.cpp index 98ba8a7b5f..1250627f8d 100644 --- a/ydb/core/mind/ut_fat/blobstorage_node_warden_ut_fat.cpp +++ b/ydb/core/mind/ut_fat/blobstorage_node_warden_ut_fat.cpp @@ -191,7 +191,7 @@ void SetupServices(TTestActorRuntime &runtime) { static_cast<IPDiskServiceFactory*>(new TStrandedPDiskServiceFactory(runtime)) : static_cast<IPDiskServiceFactory*>(new TRealPDiskServiceFactory()))); // nodeWardenConfig->Monitoring = monitoring; - google::protobuf::TextFormat::ParseFromString(staticConfig, &nodeWardenConfig->ServiceSet); + google::protobuf::TextFormat::ParseFromString(staticConfig, nodeWardenConfig->BlobStorageConfig.MutableServiceSet()); app.SetKeyForNode(keyfile, nodeIndex); ObtainTenantKey(&nodeWardenConfig->TenantKey, app.Keys[nodeIndex]); @@ -200,7 +200,7 @@ void SetupServices(TTestActorRuntime &runtime) { if (nodeIndex == 0) { static TTempDir tempDir; TString pDiskPath = tempDir() + "/pdisk0.dat"; - nodeWardenConfig->ServiceSet.MutablePDisks(0)->SetPath(pDiskPath); + nodeWardenConfig->BlobStorageConfig.MutableServiceSet()->MutablePDisks(0)->SetPath(pDiskPath); ui64 pDiskGuid = 1; static ui64 iteration = 0; diff --git a/ydb/core/protos/blobstorage_distributed_config.proto b/ydb/core/protos/blobstorage_distributed_config.proto index c100794d2b..565d74dc4f 100644 --- a/ydb/core/protos/blobstorage_distributed_config.proto +++ b/ydb/core/protos/blobstorage_distributed_config.proto @@ -13,7 +13,7 @@ message TStorageConfig { // contents of storage metadata // Attach sender node to the recipient one; if already bound, then just update configuration. message TEvNodeConfigPush { bool Initial = 1; // set to true if this push is initial connection establishment - TStorageConfig StorageConfig = 2; // configuration update (if called for the first time) + reserved 2; repeated uint32 NewBoundNodeIds = 3; // a list of nodes (not including sender) that are attached repeated uint32 DeletedBoundNodeIds = 4; // a list of detached nodes } @@ -28,3 +28,41 @@ message TEvNodeConfigReversePush { // Remove node from bound list. message TEvNodeConfigUnbind { } + +// Propagate query to the tree bottom and collect replies. +message TEvNodeConfigScatter { + message TCollectConfigs {} + + message TApplyConfigs { + optional TStorageConfig Config = 1; // config to apply + } + + optional uint64 Cookie = 1; + + oneof Request { + TCollectConfigs CollectConfigs = 2; + TApplyConfigs ApplyConfigs = 3; + } +} + +// Collected replies from the bottom. +message TEvNodeConfigGather { + message TCollectConfigs { + message TItem { + repeated uint32 NodeIds = 1; // node ids with the same config (generation & fingerprint) + optional TStorageConfig Config = 2; + } + repeated TItem Items = 1; + } + + message TApplyConfigs { + repeated uint32 NodeIds = 1; // node ids that have processed this config + } + + optional uint64 Cookie = 1; + + oneof Response { + TCollectConfigs CollectConfigs = 2; + TApplyConfigs ApplyConfigs = 3; + } +} diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 84f0b9cd9f..4a6b94c04d 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -272,7 +272,25 @@ message TBlobStorageConfig { optional string CacheFilePath = 3; optional bool CachePDisks = 4 [default = true]; optional bool CacheVDisks = 5 [default = true]; -} + + // Automatic configuration: when not filled in, then the autoconfig feature is disabled. When multiple nodes are + // negotiating about this feature, the disabled one overrides others. + message TAutoconfigSettings { + // subset of DefineStoragePool command + optional string ErasureSpecies = 1; + optional NKikimrBlobStorage.TGroupGeometry Geometry = 2; + repeated NKikimrBlobStorage.TPDiskFilter PDiskFilter = 3; + + // some extra settings + optional bool AutomaticBoxManagement = 4; // invoke BSC DefineHostConfig/DefineBox automatically + + // filled in by config parser, not by user; required for automatic static group creation + repeated NKikimrBlobStorage.TDefineHostConfig DefineHostConfig = 5; + optional NKikimrBlobStorage.TDefineBox DefineBox = 6; + } + + optional TAutoconfigSettings AutoconfigSettings = 6; + } message TBlobStorageFormatConfig { message TDrive { diff --git a/ydb/core/testlib/basics/storage.h b/ydb/core/testlib/basics/storage.h index 80878f54da..f86117018a 100644 --- a/ydb/core/testlib/basics/storage.h +++ b/ydb/core/testlib/basics/storage.h @@ -88,10 +88,10 @@ namespace NKikimr { { auto text = MakeTextConf(domains); - google::protobuf::TextFormat::ParseFromString(text, &conf->ServiceSet); + google::protobuf::TextFormat::ParseFromString(text, conf->BlobStorageConfig.MutableServiceSet()); } - conf->ServiceSet.SetEnableProxyMock(Mock); + conf->BlobStorageConfig.MutableServiceSet()->SetEnableProxyMock(Mock); conf->PDiskConfigOverlay.SetGetDriveDataSwitch(NKikimrBlobStorage::TPDiskConfig::DoNotTouch); conf->PDiskConfigOverlay.SetWriteCacheSwitch(NKikimrBlobStorage::TPDiskConfig::DoNotTouch); diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 8c89c6fc81..6040708e92 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -834,10 +834,68 @@ namespace NKikimr::NYaml { } } + void PrepareBlobStorageConfig(NJson::TJsonValue& json) { + if (!json.Has("blob_storage_config")) { + return; + } + auto& blobStorageConfig = json["blob_storage_config"]; + + if (!blobStorageConfig.Has("autoconfig_settings")) { + return; + } + auto& autoconfigSettings = blobStorageConfig["autoconfig_settings"]; + + autoconfigSettings.EraseValue("define_host_config"); + autoconfigSettings.EraseValue("define_box"); + + if (json.Has("host_configs")) { + auto& array = autoconfigSettings.InsertValue("define_host_config", NJson::JSON_ARRAY); + for (const auto& hostConfig : json["host_configs"].GetArraySafe()) { + array.AppendValue(NJson::TJsonValue(hostConfig)); + } + } + + THashMap<std::tuple<TString, ui32>, ui32> hostNodeMap; + Y_ENSURE_BT(json.Has("nameservice_config")); + const auto& nameserviceConfig = json["nameservice_config"]; + Y_ENSURE_BT(nameserviceConfig.Has("node")); + for (const auto& item : nameserviceConfig["node"].GetArraySafe()) { + const auto key = std::make_tuple(item["interconnect_host"].GetStringSafe(), item["port"].GetUIntegerSafe()); + hostNodeMap[key] = item["node_id"].GetUIntegerSafe(); + } + + NJson::TJsonValue *defineBox = nullptr; + + if (!json.Has("hosts")) { + return; + } + for (const auto& host : json["hosts"].GetArraySafe()) { + if (host.Has("host_config_id")) { + if (!defineBox) { + defineBox = &autoconfigSettings.InsertValue("define_box", NJson::TJsonMap{ + {"box_id", 1}, + {"host", NJson::TJsonArray{}}, + }); + } + + const TString fqdn = host["interconnect_host"].GetStringSafe(host["host"].GetStringSafe()); + const ui32 port = host["port"].GetUIntegerSafe(19001); + const auto key = std::make_tuple(fqdn, port); + Y_ENSURE_BT(hostNodeMap.contains(key)); + + (*defineBox)["host"].AppendValue(NJson::TJsonMap{ + {"host_config_id", host["host_config_id"].GetUIntegerSafe()}, + {"enforced_node_id", hostNodeMap[key]}, + }); + } + } + } + void TransformConfig(NJson::TJsonValue& json, bool relaxed) { PrepareNameserviceConfig(json); PrepareActorSystemConfig(json); PrepareStaticGroup(json); + PrepareBlobStorageConfig(json); PrepareIcConfig(json); PrepareLogConfig(json); PrepareSystemTabletsInfo(json, relaxed); |